visgraph.dbcore

72 ''' 73 A DbGraphStore object may be used for all the standard management 74 of node and edge information but may not be used for path queries. 75 76 FIXME possibly make it able to do path queries but *really* slow? 77 78 Use the buildSubGraph() API to pull path serchable graphs out of 79 the DBGraphStore. 80 '''

81 - def __init__(self, dbinfo=None):

82 if dbinfo == None: 83 dbinfo = default_dbinfo 84 self.dbinfo = dbinfo 85 self.db = psycopg2.connect(**dbinfo) 86 self.autocommit = True

87

88 - def _doSelect(self, query, *args):

89 ''' 90 For now, a fetchall based select wrapper. 91 ''' 92 c = self.db.cursor() 93 try: 94 c.execute(query, args) 95 return c.fetchall() 96 finally: 97 c.close()

98

99 - def _doInsert(self, query, *args):

100 ''' 101 Standard insert wrapper. 102 ''' 103 c = self.db.cursor() 104 c.execute(query, args) 105 c.close() 106 if self.autocommit: 107 self.db.commit()

108

109 - def _doUpdate(self, query, *args):

110 ''' 111 Do an update with 'returning' syntax to know 112 if an update was made. 113 ''' 114 res = [] 115 c = self.db.cursor() 116 c.execute(query, args) 117 res = c.fetchall() 118 c.close() 119 if self.autocommit: 120 self.db.commit() 121 return res

122

123 - def _doInsertRetId(self, query, *args):

124 ''' 125 Insert with a returning value. 126 ''' 127 c = self.db.cursor() 128 c.execute(query, args) 129 hid = c.fetchall()[0][0] 130 c.close() 131 if self.autocommit: 132 self.db.commit() 133 return hid

134

135 - def _doCommit(self):

136 self.db.commit()

137

138 - def addNode(self, nodeid=None, ninfo=None, **kwargs):

139 if nodeid != None: 140 raise Exception('DbGraphStore Manages nodeid!') 141 q = 'INSERT INTO vg_nodes DEFAULT VALUES RETURNING nid' 142 nid = self._doInsertRetId(q) 143 144 if ninfo != None: 145 kwargs.update(ninfo) 146 147 for key,val in kwargs.items(): 148 self.setNodeInfo(nid, key, val) 149 150 return nid

151

152 - def delEdge(self, eid):

153 ''' 154 Delete an edge from the graph database. 155 156 Example: g.delEdge(eid) 157 ''' 158 q = ''' 159 DELETE FROM 160 vg_edge_props 161 WHERE 162 eid = %s 163 ''' 164 self._doInsert(q, eid) 165 q = ''' 166 DELETE FROM 167 vg_edges 168 WHERE 169 eid = %s 170 ''' 171 self._doInsert(q, eid)

172

173 - def delNode(self, nid):

174 ''' 175 Delete the given node (and his edges) from the graph dbase. 176 177 Example: g.delNode(nid) 178 179 NOTE: this will delete any edges which go to or from nid! 180 ''' 181 for eid,n1,n2,einfo in self.getRefsFrom(nid): 182 self.delEdge(eid) 183 for eid,n1,n2,einfo in self.getRefsTo(nid): 184 self.delEdge(eid) 185 q = ''' 186 DELETE FROM 187 vg_node_props 188 WHERE 189 nid = %s 190 ''' 191 self._doInsert(q, nid) 192 q = ''' 193 DELETE FROM 194 vg_nodes 195 WHERE 196 nid = %s 197 ''' 198 self._doInsert(q, nid)

199

200 - def setNodeInfo(self, nid, pname, value):

201 if isinstance(value, int) or isinstance(value, long): 202 q = 'UPDATE vg_node_props SET intval=%s WHERE nid=%s and pname=%s RETURNING nid' 203 q1 = 'INSERT INTO vg_node_props (nid, pname, intval) VALUES (%s,%s,%s)' 204 else: 205 q = 'UPDATE vg_node_props SET strval=%s WHERE nid=%s and pname=%s RETURNING nid' 206 q1 = 'INSERT INTO vg_node_props (nid, pname, strval) VALUES (%s,%s,%s)' 207 # return a value to see if we actually did the update... 208 res = self._doSelect(q, value, nid, pname) 209 if len(res) == 0: 210 self._doInsert(q1, nid, pname, value) 211 if self.autocommit: 212 self.db.commit()

213

214 - def getNodeInfo(self, nid, pname, default=None):

215 q = 'SELECT intval,strval from vg_node_props WHERE nid=%s AND pname=%s' 216 res = self._doSelect(q, nid, pname) 217 if len(res) == 0: 218 return default 219 intval, strval = res[0] 220 if intval != None: 221 return intval 222 return strval

223

224 - def delNodeInfo(self, nid, pname):

225 q = 'DELETE FROM vg_node_props WHERE nid=%s AND pname=%s' 226 self._doInsert(q, nid, pname)

227

228 - def getNodeProps(self, nid):

229 ret = {} 230 q = 'SELECT pname,intval,strval FROM vg_node_props WHERE nid=%s' 231 for pname,intval,strval in self._doSelect(q, nid): 232 if intval != None: 233 ret[pname] = intval 234 else: 235 ret[pname] = strval 236 return ret

237

238 - def addEdge(self, fromid, toid, eid=None, einfo=None):

239 if eid != None: 240 raise Exception('DbGraphStore Manages eid!') 241 if fromid == None: 242 raise Exception('Invalid from id (None)!') 243 if toid == None: 244 raise Exception('Invalid to id (None)!') 245 q = 'INSERT INTO vg_edges (n1, n2) VALUES (%s, %s) RETURNING eid' 246 eid = self._doInsertRetId(q, fromid, toid) 247 if einfo != None: 248 for key,val in einfo.items(): 249 self.setEdgeInfo(eid, key, val) 250 return eid

251

252 - def getRefsFrom(self, nodeid):

253 ''' 254 Return a list of edges which originate with us. 255 256 Example: for eid, fromid, toid, einfo in g.getRefsFrom(id) 257 ''' 258 q = ''' 259 SELECT 260 vg_edges.*, 261 vg_edge_props.* 262 FROM 263 vg_edges, 264 vg_edge_props 265 WHERE 266 vg_edges.n1 = %s AND 267 vg_edges.eid = vg_edge_props.eid 268 ''' 269 refs = {} 270 res = self._doSelect(q, nodeid) 271 for eid,n1,n2,created,eid1,pname,intval,strval,created1 in res: 272 r = refs.get(eid) 273 if r == None: 274 r = (eid, n1, n2, {}) 275 refs[eid] = r 276 277 if intval != None: 278 r[3][pname] = intval 279 else: 280 r[3][pname] = strval 281 282 return refs.values()

283

284 - def getRefsTo(self, nodeid):

285 ''' 286 Return a list of edges which we reference. 287 288 Example: for eid, fromid, toid, einfo in g.getRefsTo(id) 289 ''' 290 q = ''' 291 SELECT 292 vg_edges.*, 293 vg_edge_props.* 294 FROM 295 vg_edges, 296 vg_edge_props 297 WHERE 298 vg_edges.n2 = %s AND 299 vg_edges.eid = vg_edge_props.eid 300 ''' 301 refs = {} 302 res = self._doSelect(q, nodeid) 303 for eid,n1,n2,created,eid1,pname,intval,strval,created1 in res: 304 r = refs.get(eid) 305 if r == None: 306 r = (eid, n1, n2, {}) 307 refs[eid] = r 308 309 if intval != None: 310 r[3][pname] = intval 311 else: 312 r[3][pname] = strval 313 314 return refs.values()

315

316 - def setEdgeInfo(self, eid, pname, value):

317 if isinstance(value, int) or isinstance(value, long): 318 q = 'UPDATE vg_edge_props SET intval=%s WHERE eid=%s and pname=%s RETURNING eid' 319 q1 = 'INSERT INTO vg_edge_props (eid, pname, intval) VALUES (%s,%s,%s)' 320 else: 321 q = 'UPDATE vg_edge_props SET strval=%s WHERE eid=%s and pname=%s RETURNING eid' 322 q1 = 'INSERT INTO vg_edge_props (eid, pname, strval) VALUES (%s,%s,%s)' 323 # return a value to see if we actually did the update... 324 res = self._doSelect(q, value, eid, pname) 325 if len(res) == 0: 326 self._doInsert(q1, eid, pname, value)

327

328 - def getEdgeInfo(self, eid, pname, default=None):

329 q = 'SELECT intval,strval from vg_edge_props WHERE eid=%s AND pname=%s' 330 res = self._doSelect(q, eid, pname) 331 if len(res) == 0: 332 return default 333 intval, strval = res[0] 334 if intval != None: 335 return intval 336 return strval

337

338 - def searchNodes(self, propname, propval=None):

339 ''' 340 Return (but do not cache forward) the nid's of nodes which 341 have a property with the following name (and optionally, value). 342 343 Example: 344 for nid in g.searchNodes('woot', 10) 345 print g.getNodeInfo(nid, 'name') 346 347 NOTE: This is specific to the DbGraphStore... 348 ''' 349 if propval == None: 350 q = 'SELECT nid FROM vg_node_props WHERE pname=%s' 351 c = self.db.cursor() 352 c.execute(q, (propname,)) 353 for row in c: 354 yield row 355 c.close()

356

357 - def buildSubGraph(self):

358 ''' 359 Return a subgraph which may be used to populate from the DB and 360 do path searching. 361 ''' 362 return DbSubGraph(self.dbinfo)

365 366 ''' 367 A subgraph in the database is basically a forward cached instance of selected 368 nodes and edges in an in-memory graph (visgraph.graphcore.Graph). This object 369 may then be used for traditional path tracing without going back to the database. 370 371 Any modifications to graph element properties *will* be synchronized back to the 372 database backing the given subgraph. 373 ''' 374

375 - def __init__(self, dbinfo):

376 vg_graphcore.Graph.__init__(self) 377 DbGraphStore.__init__(self, dbinfo)

378

379 - def addNode(self, nodeid=None, ninfo=None, **kwargs):

380 # Do *both* 381 nid = DbGraphStore.addNode(self, nodeid=nodeid, ninfo=ninfo, **kwargs) 382 vg_graphcore.Graph.addNode(self, nodeid=nid, ninfo=None, **kwargs) 383 return nid

384

385 - def addEdge(self, fromid, toid, einfo):

386 eid = DbGraphStore.addEdge(self, fromid, toid, einfo=einfo) 387 vg_graphcore.Graph.addEdge(self, fromid, toid, eid=eid, einfo=None) 388 return eid

389

390 - def useEdges(self, **kwargs):

391 ''' 392 Pull some edges from the DbStore backing this subgraph into the actual 393 visgraph.graphcore.Graph instance so path traversal is possible. 394 ''' 395 done = {} 396 for key,val in kwargs.items(): 397 if type(val) in (int,long): 398 # FIXME is vg_edges.eid faster or vg_edge_props? 399 q = 'SELECT vg_edges.eid,n1,n2 FROM vg_edge_props,vg_edges WHERE pname=%s AND intval=%s AND vg_edges.eid=vg_edge_props.eid' 400 else: 401 q = 'SELECT vg_edges.eid,n1,n2 FROM vg_edge_props,vg_edges WHERE pname=%s AND strval=%s AND vg_edges.eid=vg_edge_props.eid' 402 for eid,n1,n2 in self._doSelect(q, key, val): 403 print 'using: %d (%d->%d)' % (eid, n1, n2) 404 done[eid] = (eid, n1, n2) 405 406 # FIXME add the nodes for these edges 407 for eid, n1, n2 in done.values(): 408 if vg_graphcore.Graph.getNode(self, n1) == None: 409 vg_graphcore.Graph.addNode(self, nodeid=n1) 410 if vg_graphcore.Graph.getNode(self, n2) == None: 411 vg_graphcore.Graph.addNode(self, nodeid=n2) 412 vg_graphcore.Graph.addEdge(self, n1, n2, eid=eid)

413

414 - def expandNode(self, nid, maxdepth=1):

415 ''' 416 Add *all* the edges (and adjacent nodes) by traversing this nodes 417 edges to the specified depth... 418 ''' 419 todo = [(nid, 0),] 420 print 'INITIAL EXPAND',nid 421 if vg_graphcore.Graph.getNode(self, nid) == None: 422 print 'EXPANDING',nid 423 vg_graphcore.Graph.addNode(self, nodeid=nid) 424 425 while len(todo): 426 nid,depth = todo.pop() 427 428 if depth > maxdepth: 429 continue 430 431 # Do expansion based on the *database* 432 q = 'SELECT eid,n2 FROM vg_edges WHERE n1=%s' 433 for eid, n2 in self._doSelect(q, nid): 434 if vg_graphcore.Graph.getNode(self, n2) == None: 435 print 'EXPANDING',n2 436 vg_graphcore.Graph.addNode(self, nodeid=n2) 437 if vg_graphcore.Graph.getEdge(self, eid) == None: 438 vg_graphcore.Graph.addEdge(self, nid, n2, eid=eid) 439 ndepth = depth+1 440 if ndepth < maxdepth: 441 todo.append((n2, ndepth))

Source Code for Module visgraph.dbcore