1.1 --- a/ambuild2/database.py	Sat Feb 01 01:39:39 2014 -0800
     1.2 +++ b/ambuild2/database.py	Sat Feb 01 16:56:01 2014 -0800
     1.3 @@ -73,12 +73,23 @@
     1.4        val varchar(255)                          \
     1.5      )",
     1.6  
     1.7 +    "insert into vars (key, val) values ('db_version', '3')",
     1.8 +
     1.9      "create index if not exists outgoing_edge on edges(outgoing)",
    1.10      "create index if not exists incoming_edge on edges(incoming)",
    1.11      "create index if not exists weak_outgoing_edge on weak_edges(outgoing)",
    1.12      "create index if not exists weak_incoming_edge on weak_edges(incoming)",
    1.13      "create index if not exists dyn_outgoing_edge on dynamic_edges(outgoing)",
    1.14      "create index if not exists dyn_incoming_edge on dynamic_edges(incoming)",
    1.15 +
    1.16 +    # The shared output table.
    1.17 +    "create table if not exists shared_outputs( \
    1.18 +      outgoing int not null,                    \
    1.19 +      incoming int not null,                    \
    1.20 +      unique (outgoing, incoming)               \
    1.21 +    )",
    1.22 +    "create index if not exists sho_outgoing_edge on shared_outputs(outgoing)",
    1.23 +    "create index if not exists sho_incoming_edge on shared_outputs(incoming)",
    1.24    ]
    1.25    for query in queries:
    1.26      cn.execute(query)
    1.27 @@ -131,7 +142,7 @@
    1.28      except:
    1.29        version = 1
    1.30  
    1.31 -    latest_version = 2
    1.32 +    latest_version = 3
    1.33      if version == latest_version:
    1.34        return
    1.35      if version > latest_version:
    1.36 @@ -144,49 +155,71 @@
    1.37      )
    1.38  
    1.39      if version == 1:
    1.40 -      queries = [
    1.41 -        "create table if not exists vars(           \
    1.42 -          key varchar(255) primary key not null,    \
    1.43 -          val varchar(255)                          \
    1.44 -        )",
    1.45 +      version = self.upgrade_to_v2()
    1.46 +
    1.47 +    if version == 2:
    1.48 +      version = self.upgrade_to_v3()
    1.49  
    1.50 -        "drop table if exists nodestmp",
    1.51 -        "drop table if exists nodesold",
    1.52 +  def upgrade_to_v2(self):
    1.53 +    queries = [
    1.54 +      "create table if not exists vars(           \
    1.55 +        key varchar(255) primary key not null,    \
    1.56 +        val varchar(255)                          \
    1.57 +      )",
    1.58  
    1.59 -        "create table nodestmp(                       \
    1.60 -          id integer primary key autoincrement,       \
    1.61 -          type varchar(4) not null,                   \
    1.62 -          stamp real not null default 0.0,            \
    1.63 -          dirty int not null default 0,               \
    1.64 -          path text,                                  \
    1.65 -          folder int,                                 \
    1.66 -          data blob                                   \
    1.67 -        )",
    1.68 -      ]
    1.69 -      for query in queries:
    1.70 -        self.cn.execute(query)
    1.71 +      "drop table if exists nodestmp",
    1.72 +      "drop table if exists nodesold",
    1.73 +
    1.74 +      "create table nodestmp(                       \
    1.75 +        id integer primary key autoincrement,       \
    1.76 +        type varchar(4) not null,                   \
    1.77 +        stamp real not null default 0.0,            \
    1.78 +        dirty int not null default 0,               \
    1.79 +        path text,                                  \
    1.80 +        folder int,                                 \
    1.81 +        data blob                                   \
    1.82 +      )",
    1.83 +    ]
    1.84 +    for query in queries:
    1.85 +      self.cn.execute(query)
    1.86  
    1.87 -      # Migrate all rows in nodes to nodestmp.
    1.88 -      query = """
    1.89 -        select id, type, stamp, dirty, path, folder, data
    1.90 -          from nodes
    1.91 -          order by id asc
    1.92 -      """
    1.93 -      ins_query = """
    1.94 -        insert into nodestmp
    1.95 -          (id, type, stamp, dirty, path, folder, data)
    1.96 -          values
    1.97 -          (?,  ?,    ?,     ?,     ?,    ?,      ?)
    1.98 -      """
    1.99 -      for row in self.cn.execute(query):
   1.100 -        self.cn.execute(ins_query, row)
   1.101 +    # Migrate all rows in nodes to nodestmp.
   1.102 +    query = """
   1.103 +      select id, type, stamp, dirty, path, folder, data
   1.104 +        from nodes
   1.105 +        order by id asc
   1.106 +    """
   1.107 +    ins_query = """
   1.108 +      insert into nodestmp
   1.109 +        (id, type, stamp, dirty, path, folder, data)
   1.110 +        values
   1.111 +        (?,  ?,    ?,     ?,     ?,    ?,      ?)
   1.112 +    """
   1.113 +    for row in self.cn.execute(query):
   1.114 +      self.cn.execute(ins_query, row)
   1.115  
   1.116 -      self.cn.execute("alter table nodes rename to nodesold")
   1.117 -      self.cn.execute("alter table nodestmp rename to nodes")
   1.118 -      self.cn.execute("drop table nodesold")
   1.119 +    self.cn.execute("alter table nodes rename to nodesold")
   1.120 +    self.cn.execute("alter table nodestmp rename to nodes")
   1.121 +    self.cn.execute("drop table nodesold")
   1.122 +    self.cn.execute("insert or replace into vars (key, val) values ('db_version', ?)", (2,))
   1.123 +    self.cn.commit()
   1.124 +    return 2
   1.125  
   1.126 -    self.cn.execute("insert or replace into vars (key, val) values ('db_version', ?)", (latest_version,))
   1.127 +  def upgrade_to_v3(self):
   1.128 +    queries = [
   1.129 +      "create table if not exists shared_outputs( \
   1.130 +        outgoing int not null,                    \
   1.131 +        incoming int not null,                    \
   1.132 +        unique (outgoing, incoming)               \
   1.133 +      )",
   1.134 +      "create index if not exists sho_outgoing_edge on shared_outputs(outgoing)",
   1.135 +      "create index if not exists sho_incoming_edge on shared_outputs(incoming)",
   1.136 +    ]
   1.137 +    for query in queries:
   1.138 +      self.cn.execute(query)
   1.139 +    self.cn.execute("insert or replace into vars (key, val) values ('db_version', ?)", (3,))
   1.140      self.cn.commit()
   1.141 +    return 3
   1.142  
   1.143    def add_folder(self, parent, path):
   1.144      assert path not in self.path_cache_
   1.145 @@ -195,12 +228,13 @@
   1.146  
   1.147      return self.add_file(nodetypes.Mkdir, path, parent)
   1.148  
   1.149 -  def add_output(self, folder_entry, path):
   1.150 +  def add_output(self, folder_entry, path, kind = nodetypes.Output):
   1.151      assert path not in self.path_cache_
   1.152      assert not os.path.isabs(path)
   1.153      assert not folder_entry or os.path.split(path)[0] == folder_entry.path
   1.154 +    assert kind == nodetypes.Output or kind == nodetypes.SharedOutput
   1.155  
   1.156 -    return self.add_file(nodetypes.Output, path, folder_entry)
   1.157 +    return self.add_file(kind, path, folder_entry)
   1.158  
   1.159    def find_or_add_source(self, path):
   1.160      node = self.query_path(path)
   1.161 @@ -310,47 +344,57 @@
   1.162    def add_weak_edge(self, from_entry, to_entry):
   1.163      query = "insert into weak_edges (outgoing, incoming) values (?, ?)"
   1.164      self.cn.execute(query, (to_entry.id, from_entry.id))
   1.165 -    if to_entry.weak_inputs:
   1.166 +    if to_entry.weak_inputs is not None:
   1.167        to_entry.weak_inputs.add(from_entry)
   1.168  
   1.169    def add_strong_edge(self, from_entry, to_entry):
   1.170      query = "insert into edges (outgoing, incoming) values (?, ?)"
   1.171      self.cn.execute(query, (to_entry.id, from_entry.id))
   1.172 -    if to_entry.strong_inputs:
   1.173 +    if to_entry.strong_inputs is not None:
   1.174        to_entry.strong_inputs.add(from_entry)
   1.175 -    if from_entry.outgoing:
   1.176 +    if from_entry.outgoing is not None:
   1.177        from_entry.outgoing.add(to_entry)
   1.178  
   1.179    def add_dynamic_edge(self, from_entry, to_entry):
   1.180      query = "insert into dynamic_edges (outgoing, incoming) values (?, ?)"
   1.181      self.cn.execute(query, (to_entry.id, from_entry.id))
   1.182 -    if to_entry.dynamic_inputs:
   1.183 +    if to_entry.dynamic_inputs is not None:
   1.184        to_entry.dynamic_inputs.add(from_entry)
   1.185 -    if from_entry.outgoing:
   1.186 +    if from_entry.outgoing is not None:
   1.187        from_entry.outgoing.add(to_entry)
   1.188  
   1.189 +  def add_shared_output_edge(self, from_entry, to_entry):
   1.190 +    # These don't factor into the DAG in any meaningful way, so we don't
   1.191 +    # cache the results or put them into edge lists.
   1.192 +    query = "insert into shared_outputs (outgoing, incoming) values (?, ?)"
   1.193 +    self.cn.execute(query, (to_entry.id, from_entry.id))
   1.194 +
   1.195    def drop_dynamic_edge(self, from_entry, to_entry):
   1.196      query = "delete from dynamic_edges where outgoing = ? and incoming = ?"
   1.197      self.cn.execute(query, (to_entry.id, from_entry.id))
   1.198 -    if to_entry.dynamic_inputs:
   1.199 +    if to_entry.dynamic_inputs is not None:
   1.200        to_entry.dynamic_inputs.remove(from_entry)
   1.201 -    if from_entry.outgoing:
   1.202 +    if from_entry.outgoing is not None:
   1.203        from_entry.outgoing.remove(to_entry)
   1.204  
   1.205    def drop_weak_edge(self, from_entry, to_entry):
   1.206      query = "delete from weak_edges where outgoing = ? and incoming = ?"
   1.207      self.cn.execute(query, (to_entry.id, from_entry.id))
   1.208 -    if to_entry.weak_inputs:
   1.209 +    if to_entry.weak_inputs is not None:
   1.210        to_entry.weak_inputs.remove(from_entry)
   1.211  
   1.212    def drop_strong_edge(self, from_entry, to_entry):
   1.213      query = "delete from edges where outgoing = ? and incoming = ?"
   1.214      self.cn.execute(query, (to_entry.id, from_entry.id))
   1.215 -    if to_entry.strong_inputs:
   1.216 +    if to_entry.strong_inputs is not None:
   1.217        to_entry.strong_inputs.remove(from_entry)
   1.218 -    if from_entry.outgoing:
   1.219 +    if from_entry.outgoing is not None:
   1.220        from_entry.outgoing.remove(to_entry)
   1.221  
   1.222 +  def drop_shared_output_edge(self, from_entry, to_entry):
   1.223 +    query = "delete from shared_outputs where outgoing = ? and incoming = ?"
   1.224 +    self.cn.execute(query, (to_entry.id, from_entry.id))
   1.225 +
   1.226    def query_node(self, id):
   1.227      if id in self.node_cache_:
   1.228        return self.node_cache_[id]
   1.229 @@ -411,6 +455,16 @@
   1.230        outgoing.add(entry)
   1.231      return outgoing
   1.232  
   1.233 +  # Find the list of shared outputs this command generates.
   1.234 +  def query_shared_outputs(self, node):
   1.235 +    # Not cached.
   1.236 +    outgoing = set()
   1.237 +    query = "select outgoing from shared_outputs where incoming = ?"
   1.238 +    for outgoing_id, in self.cn.execute(query, (node.id,)):
   1.239 +      entry = self.query_node(outgoing_id)
   1.240 +      outgoing.add(entry)
   1.241 +    return outgoing
   1.242 +
   1.243    def query_outgoing(self, node):
   1.244      if node.outgoing is not None:
   1.245        return node.outgoing
   1.246 @@ -466,6 +520,13 @@
   1.247  
   1.248      return cmd_entry
   1.249  
   1.250 +  def query_shared_commands_of(self, node):
   1.251 +    query = "select incoming from shared_outputs where outgoing = ?"
   1.252 +    commands = []
   1.253 +    for row in self.cn.execute(query, (node.id,)):
   1.254 +      commands.append(self.query_node(row[0]))
   1.255 +    return commands
   1.256 +
   1.257    def query_dynamic_inputs(self, node):
   1.258      if node.dynamic_inputs is not None:
   1.259        return node.dynamic_inputs
   1.260 @@ -543,12 +604,14 @@
   1.261        node = self.import_node(id, row)
   1.262        aggregate(node)
   1.263  
   1.264 +  # Find the list of commands that generate this shared output.
   1.265    def query_commands(self, aggregate):
   1.266      query = """
   1.267        select type, stamp, dirty, path, folder, data, id
   1.268        from nodes
   1.269        where (type != 'src' and
   1.270               type != 'out' and
   1.271 +             type != 'sho' and
   1.272               type != 'grp' and
   1.273               type != 'mkd')
   1.274      """
   1.275 @@ -578,7 +641,7 @@
   1.276      del self.node_cache_[entry.id]
   1.277  
   1.278    def drop_folder(self, entry):
   1.279 -    assert entry.type == nodetypes.Mkdir or entry.type == nodetypes.Output
   1.280 +    assert entry.type in [nodetypes.Mkdir, nodetypes.Output, nodetypes.SharedOutput]
   1.281      assert not os.path.isabs(entry.path)
   1.282  
   1.283      if os.path.exists(entry.path):
   1.284 @@ -612,7 +675,7 @@
   1.285        self.drop_entry(entry)
   1.286  
   1.287    def drop_output(self, output):
   1.288 -    assert output.type == nodetypes.Output
   1.289 +    assert output.type == nodetypes.Output or output.type == nodetypes.SharedOutput
   1.290      util.rm_path(output.path)
   1.291      self.drop_entry(output)
   1.292  
   1.293 @@ -621,6 +684,7 @@
   1.294        # Commands should never have dynamic outgoing edges, FWIW.
   1.295        assert output.type == nodetypes.Output
   1.296        self.drop_output(output)
   1.297 +    self.cn.execute("delete from shared_outputs where incoming = ?", (cmd_entry.id,))
   1.298      self.drop_entry(cmd_entry)
   1.299  
   1.300    def add_or_update_script(self, path):
   1.301 @@ -644,23 +708,34 @@
   1.302        entry = self.import_node(id, row)
   1.303        aggregate(entry)
   1.304  
   1.305 +  def query_dead_shared_outputs(self, aggregate):
   1.306 +    query = """
   1.307 +      select id from nodes
   1.308 +        where type == '{0}'
   1.309 +        and id not in (select outgoing from shared_outputs)
   1.310 +    """.format(nodetypes.SharedOutput)
   1.311 +
   1.312 +    for row in self.cn.execute(query):
   1.313 +      entry = self.query_node(row[0])
   1.314 +      aggregate(entry)
   1.315 +
   1.316    def drop_script(self, path):
   1.317      self.cn.execute("delete from reconfigure where path = ?", (path,))
   1.318  
   1.319    def drop_group(self, group):
   1.320      self.drop_entry(group)
   1.321  
   1.322 -  def change_output_to_folder(self, entry):
   1.323 -    assert entry.type == nodetypes.Output
   1.324 +  def change_to_folder(self, entry):
   1.325 +    assert entry.type == nodetypes.Output or entry.type == nodetypes.SharedOutput
   1.326      self.drop_links(entry)
   1.327      self.cn.execute("update nodes set type = 'mkd' where id = ?", (entry.id,))
   1.328      entry.type = nodetypes.Mkdir
   1.329  
   1.330 -  def change_folder_to_output(self, entry):
   1.331 -    assert entry.type == nodetypes.Mkdir
   1.332 +  def change_to_output(self, entry, kind):
   1.333 +    assert entry.type in [nodetypes.Mkdir, nodetypes.Output, nodetypes.SharedOutput]
   1.334      self.drop_links(entry)
   1.335 -    self.cn.execute("update nodes set type = 'out' where id = ?", (entry.id,))
   1.336 -    entry.type = nodetypes.Output
   1.337 +    self.cn.execute("update nodes set type = ? where id = ?", (kind, entry.id))
   1.338 +    entry.type = kind
   1.339  
   1.340    def printGraph(self):
   1.341      # Find all mkdir nodes.

     2.1 --- a/ambuild2/frontend/amb2/gen.py	Sat Feb 01 01:39:39 2014 -0800
     2.2 +++ b/ambuild2/frontend/amb2/gen.py	Sat Feb 01 16:56:01 2014 -0800
     2.3 @@ -45,7 +45,11 @@
     2.4        os.mkdir(self.cacheFolder)
     2.5      if not self.db:
     2.6        db_path = os.path.join(self.cacheFolder, 'graph')
     2.7 -      self.db = database.CreateDatabase(db_path)
     2.8 +      if not os.path.isfile(db_path):
     2.9 +        self.db = database.CreateDatabase(db_path)
    2.10 +      else:
    2.11 +        self.db = database.Database(db_path)
    2.12 +        self.db.connect()
    2.13  
    2.14      self.db.query_scripts(lambda id,path,stamp: self.old_scripts_.add(path))
    2.15      self.db.query_mkdir(lambda entry: self.old_folders_.add(entry))
    2.16 @@ -70,6 +74,8 @@
    2.17      for group in self.old_groups_:
    2.18        self.db.drop_group(group)
    2.19  
    2.20 +    self.db.query_dead_shared_outputs(lambda e: self.db.drop_output(e))
    2.21 +
    2.22      class Node:
    2.23        def __init__(self):
    2.24          self.incoming = set()
    2.25 @@ -189,13 +195,18 @@
    2.26                         util.ConsoleNormal)
    2.27            raise Exception('Refactoring error: new folder')
    2.28          entry = self.db.add_folder(parent, path)
    2.29 -      elif entry.type == nodetypes.Output:
    2.30 -        cmd_entry = self.db.query_command_of(entry)
    2.31 -        if cmd_entry not in self.old_commands_:
    2.32 -          util.con_err(util.ConsoleRed, 'Folder has the same path as an output file generated by:\n',
    2.33 -                       util.ConsoleBlue, cmd_entry.format(),
    2.34 -                       util.ConsoleNormal)
    2.35 -          raise Exception('Output has been duplicated: {0}'.format(entry.path))
    2.36 +      elif entry.type == nodetypes.Output or entry.type == nodetypes.SharedOutput:
    2.37 +        if entry.type == nodetypes.Output:
    2.38 +          cmd_entries = [self.db.query_command_of(entry)]
    2.39 +        elif entry.type == nodetypes.SharedOutput:
    2.40 +          cmd_entries = self.db.query_shared_commands_of(entry)
    2.41 +
    2.42 +        for cmd_entry in cmd_entries:
    2.43 +          if cmd_entry not in self.old_commands_:
    2.44 +            util.con_err(util.ConsoleRed, 'Folder has the same path as an output file generated by:\n',
    2.45 +                         util.ConsoleBlue, cmd_entry.format(),
    2.46 +                         util.ConsoleNormal)
    2.47 +            raise Exception('Output has been duplicated: {0}'.format(entry.path))
    2.48  
    2.49          if self.refactoring:
    2.50            util.con_err(util.ConsoleRed, 'Path "',
    2.51 @@ -205,7 +216,7 @@
    2.52            raise Exception('Refactoring error: path changed from file to folder')
    2.53  
    2.54          self.rm_list_.append(entry.path)
    2.55 -        self.db.change_output_to_folder(entry)
    2.56 +        self.db.change_to_folder(entry)
    2.57        elif entry.type == nodetypes.Mkdir:
    2.58          # We let the same folder be generated twice, so use discard, not remove.
    2.59          self.old_folders_.discard(entry)
    2.60 @@ -254,7 +265,7 @@
    2.61                   util.ConsoleNormal)
    2.62      raise Exception('illegal path component')
    2.63  
    2.64 -  def parseOutput(self, cwd_entry, path):
    2.65 +  def parseOutput(self, cwd_entry, path, kind):
    2.66      if path[-1] == os.sep or path[-1] == os.altsep or path == '.' or path == '':
    2.67        util.con_err(util.ConsoleRed, 'Path "',
    2.68                     util.ConsoleBlue, path,
    2.69 @@ -286,35 +297,61 @@
    2.70                       util.ConsoleBlue, output_path,
    2.71                       util.ConsoleNormal)
    2.72          raise Exception('Refactoring error')
    2.73 -      return self.db.add_output(folder_entry, output_path)
    2.74 +      return self.db.add_output(folder_entry, output_path, kind)
    2.75  
    2.76 -    if entry.type == nodetypes.Output:
    2.77 +    if entry.type == kind:
    2.78        return entry
    2.79  
    2.80 -    if entry.type != nodetypes.Mkdir:
    2.81 +    if entry.type == nodetypes.Mkdir:
    2.82 +      if entry not in self.old_folders_:
    2.83 +        util.con_err(util.ConsoleRed, 'A folder is being re-used as an output file: "',
    2.84 +                     util.ConsoleBlue, entry.path,
    2.85 +                     util.ConsoleRed, '"',
    2.86 +                     util.ConsoleNormal)
    2.87 +        raise Exception('Attempted to re-use a folder as generated file')
    2.88 +
    2.89 +      if self.refactoring:
    2.90 +        util.con_err(util.ConsoleRed, 'A generated folder has changed to a generated file: ',
    2.91 +                     util.ConsoleBlue, entry.path,
    2.92 +                     util.ConsoleNormal)
    2.93 +        raise Exception('Refactoring error')
    2.94 +
    2.95 +      # We keep the node in old_folders_. This should be okay, since we've
    2.96 +      # changed the type to Output now. This way we can stick to one folder
    2.97 +      # deletion routine, since it's fairly complicated.
    2.98 +    elif entry.type == nodetypes.Output:
    2.99 +      # If we're asking for a shared output, make sure we can reuse this one.
   2.100 +      input_cmd = self.db.query_command_of(entry)
   2.101 +      if input_cmd and input_cmd not in self.old_commands_:
   2.102 +        util.con_err(util.ConsoleRed, 'First defined with command: ', input_cmd.format(), util.ConsoleNormal)
   2.103 +        raise Exception('Existing output cannot be a shared output: {0}'.format(entry.path))
   2.104 +
   2.105 +      if self.refactoring:
   2.106 +        util.con_err(util.ConsoleRed, 'An output has changed to a shared output: ',
   2.107 +                     util.ConsoleBlue, entry.path,
   2.108 +                     util.ConsoleNormal)
   2.109 +        raise Exception('Refactoring error')
   2.110 +    elif entry.type == nodetypes.SharedOutput:
   2.111 +      input_cmds = self.db.query_shared_commands_of(entry)
   2.112 +      for input_cmd in input_cmds:
   2.113 +        if input_cmd not in self.old_commands_:
   2.114 +          util.con_err(util.ConsoleRed, 'A shared output cannot be specified as an normal output.',
   2.115 +                       util.ConsoleNormal)
   2.116 +          raise Exception('Existing shared output cannot be a normal output: {0}'.format(entry.path))
   2.117 +
   2.118 +      if self.refactoring:
   2.119 +        util.con_err(util.ConsoleRed, 'A shared output has changed to a normal output: ',
   2.120 +                     util.ConsoleBlue, entry.path,
   2.121 +                     util.ConsoleNormal)
   2.122 +        raise Exception('Refactoring error')
   2.123 +    else:
   2.124        util.con_err(util.ConsoleRed, 'An existing node has been specified as an output file: "',
   2.125                     util.ConsoleBlue, entry.format(),
   2.126                     util.ConsoleRed, '"',
   2.127                     util.ConsoleNormal)
   2.128        raise Exception('Attempted to re-use an incompatible node as an output')
   2.129  
   2.130 -    if entry not in self.old_folders_:
   2.131 -      util.con_err(util.ConsoleRed, 'A folder is being re-used as an output file: "',
   2.132 -                   util.ConsoleBlue, entry.path,
   2.133 -                   util.ConsoleRed, '"',
   2.134 -                   util.ConsoleNormal)
   2.135 -      raise Exception('Attempted to re-use a folder as generated file')
   2.136 -
   2.137 -    if self.refactoring:
   2.138 -      util.con_err(util.ConsoleRed, 'A generated folder has changed to a generated file: ',
   2.139 -                   util.ConsoleBlue, entry.path,
   2.140 -                   util.ConsoleNormal)
   2.141 -      raise Exception('Refactoring error')
   2.142 -
   2.143 -    # We keep the node in old_folders_. This should be okay, since we've
   2.144 -    # changed the type to Output now. This way we can stick to one folder
   2.145 -    # deletion routine, since it's fairly complicated.
   2.146 -    self.db.change_folder_to_output(entry)
   2.147 +    self.db.change_to_output(entry, kind)
   2.148      return entry
   2.149  
   2.150    def parseInput(self, context, source):
   2.151 @@ -347,7 +384,8 @@
   2.152                   util.ConsoleNormal)
   2.153      raise Exception('Tried to use non-file node as a file path')
   2.154  
   2.155 -  def addCommand(self, context, node_type, folder, data, inputs, outputs, weak_inputs=[]):
   2.156 +  def addCommand(self, context, node_type, folder, data, inputs, outputs,
   2.157 +                 weak_inputs=[], shared_outputs=[]):
   2.158      assert not folder or isinstance(folder, nodetypes.Entry)
   2.159  
   2.160      # Build the set of weak links.
   2.161 @@ -363,10 +401,11 @@
   2.162        strong_input = self.parseInput(context, strong_input)
   2.163        strong_links.add(strong_input)
   2.164  
   2.165 +    # Build the list of outputs.
   2.166      cmd_entry = None
   2.167      output_nodes = []
   2.168      for output in outputs:
   2.169 -      output_node = self.parseOutput(folder, output)
   2.170 +      output_node = self.parseOutput(folder, output, nodetypes.Output)
   2.171        output_nodes.append(output_node)
   2.172  
   2.173        input_entry = self.db.query_command_of(output_node)
   2.174 @@ -382,27 +421,65 @@
   2.175          cmd_entry = input_entry
   2.176      # end for
   2.177  
   2.178 -    must_link = set(output_nodes)
   2.179 +    # Build the list of shared outputs.
   2.180 +    shared_output_nodes = []
   2.181 +    for shared_output in shared_outputs:
   2.182 +      shared_output_node = self.parseOutput(folder, shared_output, nodetypes.SharedOutput)
   2.183 +      shared_output_nodes.append(shared_output_node)
   2.184 +
   2.185 +    output_links = set(output_nodes)
   2.186 +    shared_links = set(shared_output_nodes)
   2.187 +
   2.188 +    # There should be no duplicates in either output list. These error messages
   2.189 +    # could be better.
   2.190 +    if len(output_nodes) > len(output_links):
   2.191 +      util.con_err(util.ConsoleRed, 'The output list contains duplicate files.',
   2.192 +                   util.ConsoleNormal)
   2.193 +      raise Exception('Shared output list contains duplicate files.')
   2.194 +    if len(shared_output_nodes) > len(shared_links):
   2.195 +      util.con_err(util.ConsoleRed, 'The output list contains duplicate files.',
   2.196 +                   util.ConsoleNormal)
   2.197 +      raise Exception('Shared output list contains duplicate files.')
   2.198 +
   2.199 +    # The intersection of output_links and shared_links should be the empty set.
   2.200 +    duplicates = output_links.intersection(shared_links)
   2.201 +    if len(duplicates):
   2.202 +      bad_entry = duplicates.pop()
   2.203 +      util.con_err(util.ConsoleRed, 'An output has been duplicated as a shared output: ',
   2.204 +                   util.ConsoleBlue, bad_entry.path,
   2.205 +                   util.ConsoleNormal)
   2.206 +      raise Exception('An output has been duplicated as a shared output.')
   2.207  
   2.208      if cmd_entry:
   2.209        # Update the entry in the database.
   2.210        self.db.update_command(cmd_entry, node_type, folder, data, self.refactoring)
   2.211  
   2.212        # Disconnect any outputs that are no longer connected to this output.
   2.213 -      # It's okay to use must_link since there should never be duplicate
   2.214 +      # It's okay to use output_links since there should never be duplicate
   2.215        # outputs.
   2.216        for outgoing in self.db.query_strong_outgoing(cmd_entry):
   2.217 -        if outgoing not in must_link:
   2.218 +        if outgoing not in output_links:
   2.219            self.db.drop_strong_edge(cmd_entry, outgoing)
   2.220            self.db.drop_output(outgoing)
   2.221          else:
   2.222 -          must_link.remove(outgoing)
   2.223 +          output_links.remove(outgoing)
   2.224 +
   2.225 +      # Do the same for shared outputs. Since there is a many:1 relationship,
   2.226 +      # we can't drop shared outputs here. We save that for a cleanup step.
   2.227 +      for outgoing in self.db.query_shared_outputs(cmd_entry):
   2.228 +        if outgoing not in shared_links:
   2.229 +          self.db.drop_shared_output_edge(cmd_entry, outgoing)
   2.230 +        else:
   2.231 +          shared_links.remove(outgoing)
   2.232  
   2.233        # Remove us from the list of commands to delete.
   2.234        self.old_commands_.remove(cmd_entry)
   2.235      else:
   2.236 +      # Note that if there are no outputs, we will always add a new command,
   2.237 +      # and the old (identical) command will be deleted.
   2.238        cmd_entry = self.db.add_command(node_type, folder, data)
   2.239  
   2.240 +    # Local helper function to warn about refactoring problems.
   2.241      def refactoring_error(node):
   2.242        util.con_err(util.ConsoleRed, 'New input introduced: ',
   2.243                     util.ConsoleBlue, node.path + '\n',
   2.244 @@ -411,12 +488,16 @@
   2.245                     util.ConsoleNormal)
   2.246        raise Exception('Refactoring error: new input introduced')
   2.247  
   2.248 -    if len(must_link) and self.refactoring:
   2.249 -      refactoring_error(must_link.pop())
   2.250 +    if len(output_links) and self.refactoring:
   2.251 +      refactoring_error(output_links.pop())
   2.252 +    if len(shared_links) and self.refactoring:
   2.253 +      refactoring_error(shared_links.pop())
   2.254  
   2.255      # Connect each output.
   2.256 -    for output_node in must_link:
   2.257 +    for output_node in output_links:
   2.258        self.db.add_strong_edge(cmd_entry, output_node)
   2.259 +    for shared_output_node in shared_links:
   2.260 +      self.db.add_shared_output_edge(cmd_entry, shared_output_node)
   2.261  
   2.262      # Connect/disconnect strong inputs.
   2.263      strong_inputs = self.db.query_strong_inputs(cmd_entry)
   2.264 @@ -445,7 +526,7 @@
   2.265        self.db.drop_weak_edge(weak_input, cmd_entry)
   2.266  
   2.267      # If we got new outputs or inputs, we need to re-run the command.
   2.268 -    changed = len(must_link) + len(strong_added) + len(weak_added)
   2.269 +    changed = len(output_links) + len(strong_added) + len(weak_added)
   2.270      if changed and not cmd_entry.dirty:
   2.271        self.db.mark_dirty(cmd_entry)
   2.272  
   2.273 @@ -594,7 +675,8 @@
   2.274                        outputs,
   2.275                        folder=-1,
   2.276                        dep_type=None,
   2.277 -                      weak_inputs=[]):
   2.278 +                      weak_inputs=[],
   2.279 +                      shared_outputs=[]):
   2.280      if folder is -1:
   2.281        folder = context.localFolder
   2.282  
   2.283 @@ -620,7 +702,8 @@
   2.284        data = data,
   2.285        inputs = inputs,
   2.286        outputs = outputs,
   2.287 -      weak_inputs = weak_inputs
   2.288 +      weak_inputs = weak_inputs,
   2.289 +      shared_outputs = shared_outputs
   2.290      )
   2.291  
   2.292    def addConfigureFile(self, context, path):

     3.1 --- a/ambuild2/frontend/base_gen.py	Sat Feb 01 01:39:39 2014 -0800
     3.2 +++ b/ambuild2/frontend/base_gen.py	Sat Feb 01 16:56:01 2014 -0800
     3.3 @@ -122,7 +122,8 @@
     3.4    def AddCopy(self, source, output_path):
     3.5      return self.generator.addCopy(self, source, output_path)
     3.6  
     3.7 -  def AddCommand(self, inputs, argv, outputs, folder=-1, dep_type=None, weak_inputs=[]):
     3.8 +  def AddCommand(self, inputs, argv, outputs, folder=-1, dep_type=None, weak_inputs=[],
     3.9 +                 shared_outputs=[]):
    3.10      return self.generator.addShellCommand(
    3.11        self,
    3.12        inputs,
    3.13 @@ -130,7 +131,8 @@
    3.14        outputs,
    3.15        folder = folder,
    3.16        dep_type = dep_type,
    3.17 -      weak_inputs = weak_inputs
    3.18 +      weak_inputs = weak_inputs,
    3.19 +      shared_outputs = shared_outputs
    3.20      )
    3.21  
    3.22    def AddConfigureFile(self, path):
    3.23 @@ -244,7 +246,8 @@
    3.24    def addCopy(self, context, source, output_path):
    3.25      raise Exception('Must be implemented!')
    3.26  
    3.27 -  def addShellCommand(self, context, inputs, argv, outputs, folder=-1, dep_type=None, weak_inputs=[]):
    3.28 +  def addShellCommand(self, context, inputs, argv, outputs, folder=-1, dep_type=None,
    3.29 +                      weak_inputs=[], shared_outputs=[]):
    3.30      raise Exception('Must be implemented!')
    3.31  
    3.32    def addConfigureFile(self, context, path):

     4.1 --- a/ambuild2/nodetypes.py	Sat Feb 01 01:39:39 2014 -0800
     4.2 +++ b/ambuild2/nodetypes.py	Sat Feb 01 16:56:01 2014 -0800
     4.3 @@ -29,6 +29,11 @@
     4.4  # but they have special handling within AMBuild.
     4.5  Output = 'out'
     4.6  
     4.7 +# An output node that is modified or created by multiple commands. It cannot
     4.8 +# be used as an input. Shared outputs are linked separately from normal
     4.9 +# outputs.
    4.10 +SharedOutput = 'sho'
    4.11 +
    4.12  # An aggregate node that reduces incoming edges to other nodes.
    4.13  Group = 'grp'
    4.14  
    4.15 @@ -66,6 +71,7 @@
    4.16    Source: 'source',
    4.17    Command: 'command',
    4.18    Output: 'output',
    4.19 +  SharedOutput: 'output',
    4.20    Group: 'group',
    4.21    Mkdir: 'mkdir',
    4.22    Copy: 'copy',
    4.23 @@ -147,7 +153,7 @@
    4.24      return self.folder.path
    4.25  
    4.26    def format(self):
    4.27 -    if self.type == Source or self.type == Output:
    4.28 +    if self.type == Source or self.type == Output or self.type == SharedOutput:
    4.29        return self.path
    4.30      text = ''
    4.31      if self.type == Mkdir:

     5.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     5.2 +++ b/tests/shared_outputs/basic/AMBuildScript	Sat Feb 01 16:56:01 2014 -0800
     5.3 @@ -0,0 +1,25 @@
     5.4 +# vim: set ts=8 sts=2 tw=99 et ft=python: 
     5.5 +import os, sys
     5.6 +
     5.7 +argv = [
     5.8 +  sys.executable,
     5.9 +  os.path.join(builder.sourcePath, 'generate_header.py'),
    5.10 +  'output.h'
    5.11 +]
    5.12 +
    5.13 +shared_outputs = [
    5.14 +# Comment for reconfigure test #1.
    5.15 +  os.path.join(builder.buildFolder, 'output.h')
    5.16 +]
    5.17 +
    5.18 +# Comment all below for reconfigure test #2.
    5.19 +work_folder = builder.AddFolder('work')
    5.20 +
    5.21 +builder.AddCommand(
    5.22 +  inputs = [],
    5.23 +  argv = argv,
    5.24 +  outputs = [],
    5.25 +  folder = work_folder,
    5.26 +  shared_outputs = shared_outputs
    5.27 +)
    5.28 +

     6.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     6.2 +++ b/tests/shared_outputs/basic/configure.py	Sat Feb 01 16:56:01 2014 -0800
     6.3 @@ -0,0 +1,6 @@
     6.4 +# vim: set sts=2 ts=8 sw=2 tw=99 et:
     6.5 +import sys
     6.6 +from ambuild2 import run
     6.7 +
     6.8 +builder = run.PrepareBuild(sourcePath = sys.path[0])
     6.9 +builder.Configure()

     7.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     7.2 +++ b/tests/shared_outputs/basic/generate_header.py	Sat Feb 01 16:56:01 2014 -0800
     7.3 @@ -0,0 +1,9 @@
     7.4 +# vim: set sts=2 ts=8 sw=2 tw=99 et:
     7.5 +import sys
     7.6 +
     7.7 +with open(sys.argv[1], 'w') as fp:
     7.8 +  fp.write("""
     7.9 +#ifndef HELLO_STRING
    7.10 +# define HELLO_STRING "HELLO!"
    7.11 +#endif
    7.12 +""")

     8.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     8.2 +++ b/tests/shared_outputs/duplicates/AMBuildScript	Sat Feb 01 16:56:01 2014 -0800
     8.3 @@ -0,0 +1,40 @@
     8.4 +# vim: set ts=8 sts=2 tw=99 et ft=python: 
     8.5 +import os, sys
     8.6 +
     8.7 +argv = [
     8.8 +  sys.executable,
     8.9 +  os.path.join(builder.sourcePath, 'generate_header.py'),
    8.10 +  'output.h'
    8.11 +]
    8.12 +
    8.13 +shared_outputs = [
    8.14 +  os.path.join(builder.buildFolder, 'output.h')
    8.15 +]
    8.16 +
    8.17 +def FailTest1():
    8.18 +  builder.AddCommand(
    8.19 +    inputs = [],
    8.20 +    argv = argv,
    8.21 +    outputs = [],
    8.22 +    shared_outputs = shared_outputs + shared_outputs
    8.23 +  )
    8.24 +
    8.25 +def FailTest2():
    8.26 +  builder.AddCommand(
    8.27 +    inputs = [],
    8.28 +    argv = argv,
    8.29 +    outputs = shared_outputs + shared_outputs
    8.30 +  )
    8.31 +
    8.32 +def FailTest3():
    8.33 +  builder.AddCommand(
    8.34 +    inputs = [],
    8.35 +    argv = argv,
    8.36 +    outputs = shared_outputs,
    8.37 +    shared_outputs = shared_outputs
    8.38 +  )
    8.39 +
    8.40 +# All of these should fail.
    8.41 +FailTest1()
    8.42 +#FailTest2()
    8.43 +#FailTest3()

     9.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     9.2 +++ b/tests/shared_outputs/duplicates/configure.py	Sat Feb 01 16:56:01 2014 -0800
     9.3 @@ -0,0 +1,6 @@
     9.4 +# vim: set sts=2 ts=8 sw=2 tw=99 et:
     9.5 +import sys
     9.6 +from ambuild2 import run
     9.7 +
     9.8 +builder = run.PrepareBuild(sourcePath = sys.path[0])
     9.9 +builder.Configure()

    10.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    10.2 +++ b/tests/shared_outputs/duplicates/generate_header.py	Sat Feb 01 16:56:01 2014 -0800
    10.3 @@ -0,0 +1,9 @@
    10.4 +# vim: set sts=2 ts=8 sw=2 tw=99 et:
    10.5 +import sys
    10.6 +
    10.7 +with open(sys.argv[1], 'w') as fp:
    10.8 +  fp.write("""
    10.9 +#ifndef HELLO_STRING
   10.10 +# define HELLO_STRING "HELLO!"
   10.11 +#endif
   10.12 +""")

    11.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    11.2 +++ b/tests/shared_outputs/mixup/AMBuildScript	Sat Feb 01 16:56:01 2014 -0800
    11.3 @@ -0,0 +1,70 @@
    11.4 +# vim: set ts=8 sts=2 tw=99 et ft=python: 
    11.5 +import os, sys
    11.6 +
    11.7 +argv = [
    11.8 +  sys.executable,
    11.9 +  os.path.join(builder.sourcePath, 'generate_header.py'),
   11.10 +  'output.h'
   11.11 +]
   11.12 +
   11.13 +shared_outputs = [
   11.14 +  os.path.join(builder.buildFolder, 'output.h')
   11.15 +]
   11.16 +
   11.17 +def Test1():
   11.18 +  builder.AddCommand(
   11.19 +    inputs = [],
   11.20 +    argv = argv,
   11.21 +    outputs = [],
   11.22 +    shared_outputs = shared_outputs
   11.23 +  )
   11.24 +
   11.25 +def Test2():
   11.26 +  # Change the output to a normal output.
   11.27 +  builder.AddCommand(
   11.28 +    inputs = [],
   11.29 +    argv = argv,
   11.30 +    outputs = shared_outputs
   11.31 +  )
   11.32 +
   11.33 +def Test3():
   11.34 +  # Change back to a shared output.
   11.35 +  Test1()
   11.36 +
   11.37 +def Test4():
   11.38 +  # Change to a folder.
   11.39 +  builder.AddFolder('output.h')
   11.40 +
   11.41 +def Test5():
   11.42 +  # Change back to a shared output.
   11.43 +  Test1()
   11.44 +
   11.45 +def TestFail1():
   11.46 +  Test2()
   11.47 +  Test1()
   11.48 +
   11.49 +def TestFail2():
   11.50 +  Test1()
   11.51 +  Test2()
   11.52 +
   11.53 +def TestFail3():
   11.54 +  Test1()
   11.55 +  Test4()
   11.56 +
   11.57 +def TestFail4():
   11.58 +  Test4()
   11.59 +  Test1()
   11.60 +
   11.61 +# Uncomment 1, run, recomment, uncomment 2 to test. Repeat. Each time using
   11.62 +# refactoring mode should fail.
   11.63 +Test1()
   11.64 +#Test2()
   11.65 +#Test3()
   11.66 +#Test4()
   11.67 +#Test5()
   11.68 +
   11.69 +# These tests should all fail.
   11.70 +#TestFail1()
   11.71 +#TestFail2()
   11.72 +#TestFail3()
   11.73 +#TestFail4()

    12.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    12.2 +++ b/tests/shared_outputs/mixup/configure.py	Sat Feb 01 16:56:01 2014 -0800
    12.3 @@ -0,0 +1,6 @@
    12.4 +# vim: set sts=2 ts=8 sw=2 tw=99 et:
    12.5 +import sys
    12.6 +from ambuild2 import run
    12.7 +
    12.8 +builder = run.PrepareBuild(sourcePath = sys.path[0])
    12.9 +builder.Configure()

    13.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    13.2 +++ b/tests/shared_outputs/mixup/generate_header.py	Sat Feb 01 16:56:01 2014 -0800
    13.3 @@ -0,0 +1,9 @@
    13.4 +# vim: set sts=2 ts=8 sw=2 tw=99 et:
    13.5 +import sys
    13.6 +
    13.7 +with open(sys.argv[1], 'w') as fp:
    13.8 +  fp.write("""
    13.9 +#ifndef HELLO_STRING
   13.10 +# define HELLO_STRING "HELLO!"
   13.11 +#endif
   13.12 +""")

    14.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    14.2 +++ b/tests/shared_outputs/multiples/AMBuildScript	Sat Feb 01 16:56:01 2014 -0800
    14.3 @@ -0,0 +1,34 @@
    14.4 +# vim: set ts=8 sts=2 tw=99 et ft=python: 
    14.5 +import os, sys
    14.6 +
    14.7 +argv = [
    14.8 +  sys.executable,
    14.9 +  os.path.join(builder.sourcePath, 'generate_header.py'),
   14.10 +  'output.h'
   14.11 +]
   14.12 +
   14.13 +shared_outputs = [
   14.14 +  os.path.join(builder.buildFolder, 'output.h')
   14.15 +]
   14.16 +
   14.17 +# Comment anything below to test reconfiguring.
   14.18 +builder.AddCommand(
   14.19 +  inputs = [],
   14.20 +  argv = argv,
   14.21 +  outputs = [],
   14.22 +  shared_outputs = shared_outputs
   14.23 +)
   14.24 +
   14.25 +builder.AddCommand(
   14.26 +  inputs = [],
   14.27 +  argv = argv,
   14.28 +  outputs = [],
   14.29 +  shared_outputs = shared_outputs
   14.30 +)
   14.31 +
   14.32 +builder.AddCommand(
   14.33 +  inputs = [],
   14.34 +  argv = argv,
   14.35 +  outputs = [],
   14.36 +  shared_outputs = shared_outputs
   14.37 +)

    15.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    15.2 +++ b/tests/shared_outputs/multiples/configure.py	Sat Feb 01 16:56:01 2014 -0800
    15.3 @@ -0,0 +1,6 @@
    15.4 +# vim: set sts=2 ts=8 sw=2 tw=99 et:
    15.5 +import sys
    15.6 +from ambuild2 import run
    15.7 +
    15.8 +builder = run.PrepareBuild(sourcePath = sys.path[0])
    15.9 +builder.Configure()

    16.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    16.2 +++ b/tests/shared_outputs/multiples/generate_header.py	Sat Feb 01 16:56:01 2014 -0800
    16.3 @@ -0,0 +1,9 @@
    16.4 +# vim: set sts=2 ts=8 sw=2 tw=99 et:
    16.5 +import sys
    16.6 +
    16.7 +with open(sys.argv[1], 'w') as fp:
    16.8 +  fp.write("""
    16.9 +#ifndef HELLO_STRING
   16.10 +# define HELLO_STRING "HELLO!"
   16.11 +#endif
   16.12 +""")

    17.1 --- a/tests/staticlib/generate_header.py	Sat Feb 01 01:39:39 2014 -0800
    17.2 +++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
    17.3 @@ -1,9 +0,0 @@
    17.4 -# vim: set sts=2 ts=8 sw=2 tw=99 et:
    17.5 -import sys
    17.6 -
    17.7 -with open(sys.argv[1], 'w') as fp:
    17.8 -  fp.write("""
    17.9 -#ifndef HELLO_STRING
   17.10 -# define HELLO_STRING "HELLO!"
   17.11 -#endif
   17.12 -""")