Remove the unused generated field from the nodes table (bug 6027, r=ds).
1.1 --- a/ambuild2/builder.py Sat Feb 01 02:12:32 2014 -0600
1.2 +++ b/ambuild2/builder.py Sat Feb 01 01:39:39 2014 -0800
1.3 @@ -160,7 +160,7 @@
1.4 util.ConsoleNormal
1.5 )
1.6 return None
1.7 - return self.cx.db.add_source(path, generated=True)
1.8 + return self.cx.db.add_source(path)
1.9
1.10 def discoverEntries(self, discovered_paths):
1.11 discovered_set = set()
2.1 --- a/ambuild2/database.py Sat Feb 01 02:12:32 2014 -0600
2.2 +++ b/ambuild2/database.py Sat Feb 01 01:39:39 2014 -0800
2.3 @@ -24,6 +24,70 @@
2.4
2.5 GroupPrefix = '//group/./'
2.6
2.7 +def CreateDatabase(path):
2.8 + cn = sqlite3.connect(path)
2.9 + queries = [
2.10 + "create table if not exists nodes( \
2.11 + id integer primary key autoincrement, \
2.12 + type varchar(4) not null, \
2.13 + stamp real not null default 0.0, \
2.14 + dirty int not null default 0, \
2.15 + path text, \
2.16 + folder int, \
2.17 + data blob \
2.18 + )",
2.19 +
2.20 + # The edge table stores links that are specified by the build scripts;
2.21 + # this table is essentially immutable (except for reconfigures).
2.22 + "create table if not exists edges( \
2.23 + outgoing int not null, \
2.24 + incoming int not null, \
2.25 + unique (outgoing, incoming) \
2.26 + )",
2.27 +
2.28 + # The weak edge table stores links that are specified by build scripts,
2.29 + # but only to enforce ordering. They do not propagate damage or updates.
2.30 + "create table if not exists weak_edges( \
2.31 + outgoing int not null, \
2.32 + incoming int not null, \
2.33 + unique (outgoing, incoming) \
2.34 + )",
2.35 +
2.36 + # The dynamic edge table stores edges that are discovered as a result of
2.37 + # executing a command; for example, a |cp *| or C++ #includes.
2.38 + "create table if not exists dynamic_edges( \
2.39 + outgoing int not null, \
2.40 + incoming int not null, \
2.41 + unique (outgoing, incoming) \
2.42 + )",
2.43 +
2.44 + # List of nodes which trigger a reconfigure.
2.45 + "create table if not exists reconfigure( \
2.46 + stamp real not null default 0.0, \
2.47 + path text unique \
2.48 + )",
2.49 +
2.50 + # Extra key/values we might randomly want.
2.51 + "create table if not exists vars( \
2.52 + key varchar(255) primary key not null, \
2.53 + val varchar(255) \
2.54 + )",
2.55 +
2.56 + "create index if not exists outgoing_edge on edges(outgoing)",
2.57 + "create index if not exists incoming_edge on edges(incoming)",
2.58 + "create index if not exists weak_outgoing_edge on weak_edges(outgoing)",
2.59 + "create index if not exists weak_incoming_edge on weak_edges(incoming)",
2.60 + "create index if not exists dyn_outgoing_edge on dynamic_edges(outgoing)",
2.61 + "create index if not exists dyn_incoming_edge on dynamic_edges(incoming)",
2.62 + ]
2.63 + for query in queries:
2.64 + cn.execute(query)
2.65 + cn.commit()
2.66 +
2.67 + db = Database(path)
2.68 + db.connect()
2.69 + return db
2.70 +
2.71 class Database(object):
2.72 def __init__(self, path):
2.73 self.path = path
2.74 @@ -35,6 +99,7 @@
2.75 assert not self.cn
2.76 self.cn = sqlite3.connect(self.path)
2.77 self.cn.execute("PRAGMA journal_mode = WAL;")
2.78 + self.check_upgrade()
2.79
2.80 def close(self):
2.81 if self.cn:
2.82 @@ -55,58 +120,72 @@
2.83 self.node_cache_ = {}
2.84 self.path_cache_ = {}
2.85
2.86 - def create_tables(self):
2.87 - queries = [
2.88 - "create table if not exists nodes( \
2.89 - id integer primary key autoincrement, \
2.90 - type varchar(4) not null, \
2.91 - stamp real not null default 0.0, \
2.92 - dirty int not null default 0, \
2.93 - generated int not null default 0, \
2.94 - path text, \
2.95 - folder int, \
2.96 - data blob \
2.97 - )",
2.98 + def check_upgrade(self):
2.99 + try:
2.100 + query = "select val from vars where key = 'db_version'"
2.101 + cursor = self.cn.execute(query)
2.102 + row = cursor.fetchone()
2.103 + if not row:
2.104 + raise Exception('Database seems to be misconfigured - cannot read version')
2.105 + version = int(row[0])
2.106 + except:
2.107 + version = 1
2.108
2.109 - # The edge table stores links that are specified by the build scripts;
2.110 - # this table is essentially immutable (except for reconfigures).
2.111 - "create table if not exists edges( \
2.112 - outgoing int not null, \
2.113 - incoming int not null, \
2.114 - unique (outgoing, incoming) \
2.115 - )",
2.116 + latest_version = 2
2.117 + if version == latest_version:
2.118 + return
2.119 + if version > latest_version:
2.120 + raise Exception('Your database version is too new!')
2.121 +
2.122 + util.con_out(
2.123 + util.ConsoleHeader,
2.124 + 'Note: upgrading database from version {0} to {1}'.format(version, latest_version),
2.125 + util.ConsoleNormal
2.126 + )
2.127 +
2.128 + if version == 1:
2.129 + queries = [
2.130 + "create table if not exists vars( \
2.131 + key varchar(255) primary key not null, \
2.132 + val varchar(255) \
2.133 + )",
2.134 +
2.135 + "drop table if exists nodestmp",
2.136 + "drop table if exists nodesold",
2.137
2.138 - # The weak edge table stores links that are specified by build scripts,
2.139 - # but only to enforce ordering. They do not propagate damage or updates.
2.140 - "create table if not exists weak_edges( \
2.141 - outgoing int not null, \
2.142 - incoming int not null, \
2.143 - unique (outgoing, incoming) \
2.144 - )",
2.145 -
2.146 - # The dynamic edge table stores edges that are discovered as a result of
2.147 - # executing a command; for example, a |cp *| or C++ #includes.
2.148 - "create table if not exists dynamic_edges( \
2.149 - outgoing int not null, \
2.150 - incoming int not null, \
2.151 - unique (outgoing, incoming) \
2.152 - )",
2.153 + "create table nodestmp( \
2.154 + id integer primary key autoincrement, \
2.155 + type varchar(4) not null, \
2.156 + stamp real not null default 0.0, \
2.157 + dirty int not null default 0, \
2.158 + path text, \
2.159 + folder int, \
2.160 + data blob \
2.161 + )",
2.162 + ]
2.163 + for query in queries:
2.164 + self.cn.execute(query)
2.165
2.166 - # List of nodes which trigger a reconfigure.
2.167 - "create table if not exists reconfigure( \
2.168 - stamp real not null default 0.0, \
2.169 - path text unique \
2.170 - )",
2.171 + # Migrate all rows in nodes to nodestmp.
2.172 + query = """
2.173 + select id, type, stamp, dirty, path, folder, data
2.174 + from nodes
2.175 + order by id asc
2.176 + """
2.177 + ins_query = """
2.178 + insert into nodestmp
2.179 + (id, type, stamp, dirty, path, folder, data)
2.180 + values
2.181 + (?, ?, ?, ?, ?, ?, ?)
2.182 + """
2.183 + for row in self.cn.execute(query):
2.184 + self.cn.execute(ins_query, row)
2.185
2.186 - "create index if not exists outgoing_edge on edges(outgoing)",
2.187 - "create index if not exists incoming_edge on edges(incoming)",
2.188 - "create index if not exists weak_outgoing_edge on weak_edges(outgoing)",
2.189 - "create index if not exists weak_incoming_edge on weak_edges(incoming)",
2.190 - "create index if not exists dyn_outgoing_edge on dynamic_edges(outgoing)",
2.191 - "create index if not exists dyn_incoming_edge on dynamic_edges(incoming)",
2.192 - ]
2.193 - for query in queries:
2.194 - self.cn.execute(query)
2.195 + self.cn.execute("alter table nodes rename to nodesold")
2.196 + self.cn.execute("alter table nodestmp rename to nodes")
2.197 + self.cn.execute("drop table nodesold")
2.198 +
2.199 + self.cn.execute("insert or replace into vars (key, val) values ('db_version', ?)", (latest_version,))
2.200 self.cn.commit()
2.201
2.202 def add_folder(self, parent, path):
2.203 @@ -114,15 +193,14 @@
2.204 assert not os.path.isabs(path)
2.205 assert os.path.normpath(path) == path
2.206
2.207 - # We don't use the generated bit for folders right now.
2.208 - return self.add_file(nodetypes.Mkdir, path, False, parent)
2.209 + return self.add_file(nodetypes.Mkdir, path, parent)
2.210
2.211 def add_output(self, folder_entry, path):
2.212 assert path not in self.path_cache_
2.213 assert not os.path.isabs(path)
2.214 assert not folder_entry or os.path.split(path)[0] == folder_entry.path
2.215
2.216 - return self.add_file(nodetypes.Output, path, False, folder_entry)
2.217 + return self.add_file(nodetypes.Output, path, folder_entry)
2.218
2.219 def find_or_add_source(self, path):
2.220 node = self.query_path(path)
2.221 @@ -132,22 +210,22 @@
2.222
2.223 return self.add_source(path)
2.224
2.225 - def add_source(self, path, generated=False):
2.226 + def add_source(self, path):
2.227 assert path not in self.path_cache_
2.228 assert os.path.isabs(path)
2.229
2.230 - return self.add_file(nodetypes.Source, path, generated)
2.231 + return self.add_file(nodetypes.Source, path)
2.232
2.233 - def add_file(self, type, path, generated, folder_entry = None):
2.234 + def add_file(self, type, path, folder_entry = None):
2.235 if folder_entry:
2.236 folder_id = folder_entry.id
2.237 else:
2.238 folder_id = None
2.239
2.240 - query = "insert into nodes (type, generated, path, folder) values (?, ?, ?, ?)"
2.241 + query = "insert into nodes (type, path, folder) values (?, ?, ?)"
2.242
2.243 - cursor = self.cn.execute(query, (type, int(generated), path, folder_id))
2.244 - row = (type, 0, 1, 1, path, folder_entry, None)
2.245 + cursor = self.cn.execute(query, (type, path, folder_id))
2.246 + row = (type, 0, 1, path, folder_entry, None)
2.247 return self.import_node(
2.248 id=cursor.lastrowid,
2.249 row=row
2.250 @@ -224,8 +302,7 @@
2.251 blob=data,
2.252 folder=folder,
2.253 stamp=0,
2.254 - dirty=True,
2.255 - generated=False
2.256 + dirty=True
2.257 )
2.258 self.node_cache_[entry.id] = entry
2.259 return entry
2.260 @@ -278,7 +355,7 @@
2.261 if id in self.node_cache_:
2.262 return self.node_cache_[id]
2.263
2.264 - query = "select type, stamp, dirty, generated, path, folder, data from nodes where id = ?"
2.265 + query = "select type, stamp, dirty, path, folder, data from nodes where id = ?"
2.266 cursor = self.cn.execute(query, (id,))
2.267 return self.import_node(id, cursor.fetchone())
2.268
2.269 @@ -287,7 +364,7 @@
2.270 return self.path_cache_[path]
2.271
2.272 query = """
2.273 - select type, stamp, dirty, generated, path, folder, data, id
2.274 + select type, stamp, dirty, path, folder, data, id
2.275 from nodes
2.276 where path = ?
2.277 """
2.278 @@ -296,30 +373,29 @@
2.279 if not row:
2.280 return None
2.281
2.282 - return self.import_node(row[7], row)
2.283 + return self.import_node(row[6], row)
2.284
2.285 def import_node(self, id, row):
2.286 assert id not in self.node_cache_
2.287
2.288 - if not row[5]:
2.289 + if not row[4]:
2.290 folder = None
2.291 - elif type(row[5]) is nodetypes.Entry:
2.292 - folder = row[5]
2.293 + elif type(row[4]) is nodetypes.Entry:
2.294 + folder = row[4]
2.295 else:
2.296 - folder = self.query_node(row[5])
2.297 - if not row[6]:
2.298 + folder = self.query_node(row[4])
2.299 + if not row[5]:
2.300 blob = None
2.301 else:
2.302 - blob = util.Unpickle(row[6])
2.303 + blob = util.Unpickle(row[5])
2.304
2.305 node = Entry(id=id,
2.306 type=row[0],
2.307 - path=row[4],
2.308 + path=row[3],
2.309 blob=blob,
2.310 folder=folder,
2.311 stamp=row[1],
2.312 - dirty=row[2],
2.313 - generated=bool(row[3]))
2.314 + dirty=row[2])
2.315 self.node_cache_[id] = node
2.316 if node.path:
2.317 assert node.path not in self.path_cache_
2.318 @@ -431,25 +507,25 @@
2.319 # Query all mkdir nodes.
2.320 def query_mkdir(self, aggregate):
2.321 query = """
2.322 - select type, stamp, dirty, generated, path, folder, data, id
2.323 + select type, stamp, dirty, path, folder, data, id
2.324 from nodes
2.325 where type == 'mkd'
2.326 """
2.327 for row in self.cn.execute(query):
2.328 - id = row[7]
2.329 + id = row[6]
2.330 node = self.import_node(id, row)
2.331 aggregate(node)
2.332
2.333 # Intended to be called before any nodes are imported.
2.334 def query_known_dirty(self, aggregate):
2.335 query = """
2.336 - select type, stamp, dirty, generated, path, folder, data, id
2.337 + select type, stamp, dirty, path, folder, data, id
2.338 from nodes
2.339 where dirty = 1
2.340 and type != 'mkd'
2.341 """
2.342 for row in self.cn.execute(query):
2.343 - id = row[7]
2.344 + id = row[6]
2.345 node = self.import_node(id, row)
2.346 aggregate(node)
2.347
2.348 @@ -457,19 +533,19 @@
2.349 # be called after query_dirty, and returns a mutually exclusive list.
2.350 def query_maybe_dirty(self, aggregate):
2.351 query = """
2.352 - select type, stamp, dirty, generated, path, folder, data, id
2.353 + select type, stamp, dirty, path, folder, data, id
2.354 from nodes
2.355 where dirty = 0
2.356 and (type == 'src' or type == 'out' or type == 'cpa')
2.357 """
2.358 for row in self.cn.execute(query):
2.359 - id = row[7]
2.360 + id = row[6]
2.361 node = self.import_node(id, row)
2.362 aggregate(node)
2.363
2.364 def query_commands(self, aggregate):
2.365 query = """
2.366 - select type, stamp, dirty, generated, path, folder, data, id
2.367 + select type, stamp, dirty, path, folder, data, id
2.368 from nodes
2.369 where (type != 'src' and
2.370 type != 'out' and
2.371 @@ -477,7 +553,7 @@
2.372 type != 'mkd')
2.373 """
2.374 for row in self.cn.execute(query):
2.375 - id = row[7]
2.376 + id = row[6]
2.377 node = self.import_node(id, row)
2.378 aggregate(node)
2.379
2.380 @@ -559,12 +635,12 @@
2.381
2.382 def query_groups(self, aggregate):
2.383 query = """
2.384 - select type, stamp, dirty, generated, path, folder, data, id
2.385 + select type, stamp, dirty, path, folder, data, id
2.386 from nodes
2.387 where type == 'grp'
2.388 """
2.389 for row in self.cn.execute(query):
2.390 - id = row[7]
2.391 + id = row[6]
2.392 entry = self.import_node(id, row)
2.393 aggregate(entry)
2.394
3.1 --- a/ambuild2/frontend/amb2/gen.py Sat Feb 01 02:12:32 2014 -0600
3.2 +++ b/ambuild2/frontend/amb2/gen.py Sat Feb 01 01:39:39 2014 -0800
3.3 @@ -44,9 +44,8 @@
3.4 if not os.path.isdir(self.cacheFolder):
3.5 os.mkdir(self.cacheFolder)
3.6 if not self.db:
3.7 - self.db = database.Database(os.path.join(self.cacheFolder, 'graph'))
3.8 - self.db.connect()
3.9 - self.db.create_tables()
3.10 + db_path = os.path.join(self.cacheFolder, 'graph')
3.11 + self.db = database.CreateDatabase(db_path)
3.12
3.13 self.db.query_scripts(lambda id,path,stamp: self.old_scripts_.add(path))
3.14 self.db.query_mkdir(lambda entry: self.old_folders_.add(entry))
4.1 --- a/ambuild2/nodetypes.py Sat Feb 01 02:12:32 2014 -0600
4.2 +++ b/ambuild2/nodetypes.py Sat Feb 01 01:39:39 2014 -0800
4.3 @@ -90,7 +90,7 @@
4.4
4.5 # The basic properties of a node as it exists in the database.
4.6 class Entry(object):
4.7 - def __init__(self, id, type, path, blob, folder, stamp, dirty, generated):
4.8 + def __init__(self, id, type, path, blob, folder, stamp, dirty):
4.9 # Unique node ID (integer)
4.10 self.id = id
4.11
4.12 @@ -121,10 +121,6 @@
4.13 # 2 if the node has become dirty in the meantime.
4.14 self.dirty = dirty
4.15
4.16 - # 1 if the node was automatically generated by the build process, 0
4.17 - # otherwise.
4.18 - self.generated = generated
4.19 -
4.20 #########################################
4.21 # Remaining fields are lazily computed. #
4.22 #########################################
