diff --git a/SCRAM/BuildSystem/BuildFile.py b/SCRAM/BuildSystem/BuildFile.py index 07be087..6ed0520 100644 --- a/SCRAM/BuildSystem/BuildFile.py +++ b/SCRAM/BuildSystem/BuildFile.py @@ -1,5 +1,5 @@ from SCRAM import printerror, scramerror -from SCRAM.BuildSystem.SimpleDoc import SimpleDoc +from SCRAM.BuildSystem.SimpleDoc import SimpleDoc, replaceVariables, loopData from SCRAM.BuildSystem.TemplateStash import TemplateStash from os.path import basename from json import dump @@ -17,7 +17,7 @@ def __init__(self, toolmanager=None, contents={}): self.flags = {} self.selected = {} self.group = [] - self.loop_products = [] + self.product = None self.variables = TemplateStash() self.toolmanager = toolmanager self.parser = SimpleDoc() @@ -115,87 +115,35 @@ def _clean(self, filename=None): self.filename = filename self.flags = {} self.selected = {} - self.loop_products = [] self.group = [] self.variables = TemplateStash() self.contents = {'USE': [], 'EXPORT': {}, 'FLAGS': {}, 'BUILDPRODUCTS': {}} def _update_product(self, tag, value, key=None): - for (prod,index) in self.loop_products if self.loop_products else [(self.product,None)]: - if tag not in prod: - prod[tag] = [] if key is None else {} - pre_data = {} if index is None else {"value": index} - if key is None: - prod[tag].append(self._replace_variables(value, pre_data)) - else: - key = self._replace_variables(key, pre_data) - if key not in prod[tag]: - prod[tag][key] = [] - prod[tag][key].append(self._replace_variables(value, pre_data)) + if tag not in self.product: + self.product[tag] = [] if key is None else {} + if key is None: + self.product[tag].append(replaceVariables(value, self.variables)) + else: + key = replaceVariables(key, self.variables) + if key not in self.product[tag]: + self.product[tag][key] = [] + self.product[tag][key].append(replaceVariables(value, self.variables)) return - def _check_value(self, data): - if search('[$][(]+[^)]+\\s', data) or search('[$][{]+[^}]+\\s', data): - scramerror("Invalid attribute value '%s' found for tag '%s' in %s." % (data, self.tag, self.filename)) - return data - - def _replace_variables(self, data, pre_data, recursive=False): - if not data: return data - m = reReplaceEnv.match(data) - if not m: return self._check_value(data) - value = pre_data[m.group(3)] if (m.group(3) in pre_data) else self.variables.get(m.group(3), default=None) - value = m.group(2) if (value is None) else self._replace_variables(value, pre_data, recursive=True) - xdata = "%s%s%s" % (self._replace_variables(m.group(1), pre_data, recursive=True), \ - value, \ - self._replace_variables(m.group(4), pre_data, recursive=True)) - data = data if (xdata == data) else self._replace_variables(xdata, pre_data, recursive=True) - if not recursive: - data = self._check_value(data) - return data - - def _add_loop_products(self, data, tag_name, prod_type): - loop_data = [] - if 'for' in data.attrib: - loops_vals = data.attrib['for'].split(",", 2) - loop_items = [1, int(loops_vals[-1]), 1] - if len(loops_vals)>1: - loop_items[0] = int(loops_vals[0]) - if len(loops_vals)>2: - loop_items[2] = loop_items[1] - loop_items[1] = int(loops_vals[1]) - self.variables.set('step_value', str(loop_items[2])) - self.variables.set('start_value', str(loop_items[0])) - self.variables.set('end_value', str(loop_items[1])) - loop_items[1] += loop_items[2] - loop_data = [str(x) for x in range(*loop_items)] - elif 'foreach' in data.attrib: - for item in [x.strip() for x in data.attrib['foreach'].split(",")]: - if (not item) or (not match('^[a-zA-Z0-9_.+-]+$', item)): - scramerror("ERROR: Invalid 'foreach' item '%s' found in file %s.\n%s" % (item, self.filename, ET.tostring(data))) - else: - loop_data.append(item) - if not loop_data: - loop_data = [""] + def _add_product(self, data, tag_name, prod_type): tag = 'BIN' if tag_name=='TEST' else tag_name if tag not in self.contents['BUILDPRODUCTS']: self.contents['BUILDPRODUCTS'][tag] = {} - xname = data.attrib['name'] if ((tag_name == 'TEST') or ('name' in data.attrib)) \ + name = data.attrib['name'] if ((tag_name == 'TEST') or ('name' in data.attrib)) \ else basename(data.attrib['file']).rsplit('.', 1)[0] - pre_data = {} - for value in loop_data: - name = xname - if value: - pre_data['value'] = value - name = "%s_%s" % (xname, value) - self.contents['BUILDPRODUCTS'][tag][name] = {'USE': [], 'EXPORT': {}, 'FLAGS': {}} - self.product = self.contents['BUILDPRODUCTS'][tag][name] - self.product['TYPE'] = prod_type - if tag_name == 'TEST': - self.product['COMMAND'] = self._replace_variables(data.attrib['command'], pre_data) - else: - self.product['FILES'] = self._replace_variables(data.attrib['file'], pre_data) - if value: - self.loop_products.append((self.product,value)) + self.contents['BUILDPRODUCTS'][tag][name] = {'USE': [], 'EXPORT': {}, 'FLAGS': {}} + self.product = self.contents['BUILDPRODUCTS'][tag][name] + self.product['TYPE'] = prod_type + if tag_name == 'TEST': + self.product['COMMAND'] = replaceVariables(data.attrib['command'], self.variables) + else: + self.product['FILES'] = replaceVariables(data.attrib['file'], self.variables) return def _update_contents(self, data): @@ -242,12 +190,11 @@ def _update_contents(self, data): self.contents[tag] = {'LIB': []} self.product = self.contents[tag] elif tag in ['BIN', 'LIBRARY', 'TEST']: - self.loop_products = [] self.variables.pushstash() if tag == 'TEST': - self._add_loop_products(data, tag, 'test') + self._add_product(data, tag, 'test') else: - self._add_loop_products(data, tag, 'bin' if tag == 'BIN' else 'lib') + self._add_product(data, tag, 'bin' if tag == 'BIN' else 'lib') elif tag == 'SET': self.variables.set(data.attrib['name'], data.attrib['value']) elif tag in ['ROOT', 'ENVIRONMENT'] or self.parser.has_filter(data.tag): @@ -268,12 +215,10 @@ def _update_contents(self, data): if not self._update_contents(child): return False if tag in ['BIN', 'LIBRARY', 'TEST']: - for prod,index in self.loop_products if self.loop_products else [(self.product,None)]: - for key in list(prod): - if not prod[key]: - del prod[key] - self.loop_products = [] self.variables.popstash() + for key in list(self.product): + if not self.product[key]: + del self.product[key] self.product = self.contents elif tag in ["EXPORT"]: self.product = self.contents diff --git a/SCRAM/BuildSystem/SimpleDoc.py b/SCRAM/BuildSystem/SimpleDoc.py index 6d0d406..94fb66c 100644 --- a/SCRAM/BuildSystem/SimpleDoc.py +++ b/SCRAM/BuildSystem/SimpleDoc.py @@ -1,9 +1,13 @@ import xml.etree.ElementTree as ET from os import environ -from re import search +from re import compile, match, search from sys import platform from platform import machine +from copy import deepcopy from SCRAM import printerror +from SCRAM.BuildSystem.TemplateStash import TemplateStash + +reReplaceEnv = compile(r'^(.*)(\$\{(\w+)\})(.*)$') DEFAULT_ENV_FILTERS = { 'ifarchitecture': 'SCRAM_ARCH', @@ -18,6 +22,41 @@ 'ifscram': 'SCRAM_VERSION' } +def replaceVariables(data, variables): + while '${' in data: + m = reReplaceEnv.match(data) + if not m: return data + value = variables.get(m.group(3), default=None) + value = m.group(2) if (value is None) else replaceVariables(value, variables) + xdata = ( + replaceVariables(m.group(1), variables) + + value + + replaceVariables(m.group(4), variables) + ) + if xdata == data: return data + data = xdata + return data + +def loopData(tag, var, value, stash): + values = [] + if tag == "foreach": + values = [replaceVariables(v.strip(), stash) + for v in value.split(",") if v.strip()] + elif tag == "for": + loops_vals = [v.strip() for v in value.split(",", 2)] + loop_items = [1, int(loops_vals[-1]), 1] + if len(loops_vals)>1: + loop_items[0] = int(loops_vals[0]) + if len(loops_vals)>2: + loop_items[2] = loop_items[1] + loop_items[1] = int(loops_vals[1]) + stash.set('step_'+var, str(loop_items[2])) + stash.set('start_'+var, str(loop_items[0])) + stash.set('end_'+var, str(loop_items[1])) + loop_items[1] += loop_items[2] + values = [str(x) for x in range(*loop_items)] + return values + class SimpleDoc(object): def __init__(self, valid_attribs={}): @@ -30,6 +69,8 @@ def __init__(self, valid_attribs={}): "library": ["name", "file", "for", "foreach"], "test": ["name", "command", "for", "foreach"], "set": ["name", "value"], + "foreach": ["set", "value"], + "for": ["set", "value"], "environment": [], "ifarchitecture": ["name", "match", "value"], "compiler": ["name", "match", "value"], @@ -60,6 +101,7 @@ def __init__(self, valid_attribs={}): self.callbacks = {} self.last_filter = [] self.filename = None + self.variables = TemplateStash() self.add_filter('ifos', platform) self.add_filter('ifarch', machine()) for filt in DEFAULT_ENV_FILTERS: @@ -149,9 +191,69 @@ def parse(self, filename): for i in range(max(0, lineno - 3), min(len(lines), lineno + 2)): print(f"{i+1}: {lines[i]}") printerror("ERROR:\n%s" % e) + self._expand(root) self.process(root) return root + def _expand_product(self, node, child): + if not child.tag.upper() in ['BIN', 'LIBRARY', 'TEST']: return False + loop_type = "" + if 'for' in child.attrib: + loop_type = "for" + elif 'foreach' in child.attrib: + loop_type = "foreach" + if not loop_type: return False + self.variables.pushstash() + var = "value" + values = loopData(loop_type, var, child.attrib[loop_type], self.variables) + name = child.attrib['name'] if 'name' in child.attrib else basename(child.attrib['file']).rsplit('.', 1)[0] + del child.attrib[loop_type] + idx = list(node).index(child) + for value in values: + self.variables.set(var, value) + new_node = deepcopy(child) + new_node.attrib['name'] = '%s_%s' % (name, value) + self._substitute(new_node) + node.insert(idx, new_node) + idx += 1 + node.remove(child) + self.variables.popstash() + return True + + def _expand(self, node): + for child in list(node): + if self._expand_product(node, child): continue + self._expand(child) + values = [] + var = None + if child.tag in ["foreach", "for"]: + self.variables.pushstash() + var = child.attrib["set"] + values = loopData(child.tag, var, child.attrib["value"], self.variables) + else: + continue + idx = list(node).index(child) + for value in values: + self.variables.set(var, value) + for grandchild in child: + new_node = deepcopy(grandchild) + self._substitute(new_node) + node.insert(idx, new_node) + idx += 1 + node.remove(child) + self.variables.popstash() + + def _substitute(self, node): + for k in list(node.attrib.keys()): + nk = replaceVariables(k, self.variables) + nv = replaceVariables(node.attrib[k], self.variables) + if k != nk: + del node.attrib[k] + node.attrib[nk] = nv + for child in node: + self._substitute(child) + return + def process(self, root): keep = True filtered = False