From 696e9dc0e71e55ff865da7516375282c5c0a3128 Mon Sep 17 00:00:00 2001 From: Kelly Boothby Date: Thu, 23 Jul 2020 16:17:33 -0700 Subject: [PATCH] busclique integration testing added integration tests for the busclique submodule * fixed several bugs in busclique c++ * corrected issue where clear_all_caches wasn't removing directories * addressed a -Werror=format-security issue for printing messages with no arguments --- include/busclique/biclique_cache.hpp | 21 ++- include/busclique/clique_cache.hpp | 4 +- include/busclique/find_clique.hpp | 2 +- include/find_embedding/util.hpp | 11 +- minorminer/busclique.pyx | 14 +- minorminer/tests/test_busclique.py | 267 +++++++++++++++++++++++++++ tests/requirements.txt | 2 +- 7 files changed, 307 insertions(+), 14 deletions(-) create mode 100644 minorminer/tests/test_busclique.py diff --git a/include/busclique/biclique_cache.hpp b/include/busclique/biclique_cache.hpp index 76df4296..0bb054c9 100644 --- a/include/busclique/biclique_cache.hpp +++ b/include/busclique/biclique_cache.hpp @@ -43,7 +43,7 @@ class yieldcache { template class biclique_cache { public: - biclique_cache(const biclique_cache&) = delete; + biclique_cache(const biclique_cache&) = delete; biclique_cache(biclique_cache &&) = delete; const cell_cache &cells; private: @@ -101,7 +101,7 @@ class biclique_cache { for(size_t x0 = 0; x0 <= cells.topo.dim[1]-w; x0++) next.set(y0, x0, 0, bundles.get_line_score(0, x0, y0, y0+h-1)); } - for(size_t w = 2; w < cells.topo.dim[1]; w++) { + for(size_t w = 2; w <= cells.topo.dim[1]; w++) { yieldcache prev = get(h, w-1); yieldcache next = get(h, w); for(size_t y0 = 0; y0 <= cells.topo.dim[0]-h; y0++) { @@ -124,7 +124,7 @@ class biclique_cache { for(size_t x0 = 0; x0 <= cells.topo.dim[1]-w; x0++) next.set(y0, x0, 1, bundles.get_line_score(1, y0, x0, x0+w-1)); } - for(size_t h = 2; h < cells.topo.dim[0]; h++) { + for(size_t h = 2; h <= cells.topo.dim[0]; h++) { yieldcache prev = get(h-1, w); yieldcache next = get(h, w); for(size_t x0 = 0; x0 <= cells.topo.dim[1]-w; x0++) { @@ -177,13 +177,20 @@ class biclique_yield_cache { vector> biclique_bounds; public: + biclique_yield_cache(const biclique_yield_cache&) = delete; + biclique_yield_cache(biclique_yield_cache &&) = delete; + biclique_yield_cache(const cell_cache &c, const bundle_cache &b, const biclique_cache &bicliques) : cells(c), bundles(b), - rows(cells.topo.dim[0]*cells.topo.shore), - cols(cells.topo.dim[1]*cells.topo.shore), + + //note: the role of rows and columns is reversed, because the indices + //are chainlengths in a given direction (not the number of chains) + rows(cells.topo.dim[1]*cells.topo.shore), + cols(cells.topo.dim[0]*cells.topo.shore), + chainlength(rows, vector(cols, 0)), biclique_bounds(rows, vector(cols, bound_t(0,0,0,0))) { compute_cache(bicliques); @@ -198,6 +205,8 @@ class biclique_yield_cache { size_t s0 = cache.get(y, x, 0); size_t s1 = cache.get(y, x, 1); if (s0 == 0 || s1 == 0) continue; + minorminer_assert(s0-1 < rows); + minorminer_assert(s1-1 < cols); size_t maxlen = cells.topo.biclique_length(y, y+h-1, x, x+w-1); size_t prevlen = chainlength[s0-1][s1-1]; if(prevlen == 0 || prevlen > maxlen) { @@ -227,7 +236,7 @@ class biclique_yield_cache { return true; } public: - iterator(size_t _s0, size_t _s1, const size_t r, const size_t c, + iterator(size_t _s0, size_t _s1, const size_t &r, const size_t &c, const vector> &cl, const vector> &_bounds, const bundle_cache &_bundles) : diff --git a/include/busclique/clique_cache.hpp b/include/busclique/clique_cache.hpp index c2070235..10ec07f0 100644 --- a/include/busclique/clique_cache.hpp +++ b/include/busclique/clique_cache.hpp @@ -102,6 +102,8 @@ class clique_cache { bundles(b), width(w), mem(new size_t[memsize()]{}) { + minorminer_assert(width <= cells.topo.dim[0]); + minorminer_assert(width <= cells.topo.dim[1]); mem[0] = width; for(size_t i = 1; i < width; i++) mem[i] = mem[i-1] + memsize(i-1); @@ -399,7 +401,7 @@ class clique_yield_cache { } stop_w1_scan:; } - for(size_t w = 2; w <= cells.topo.dim[0]; w++) { + for(size_t w = 2; w <= min(cells.topo.dim[0], cells.topo.dim[1]); w++) { size_t min_length, max_length; get_length_range(bundles, w, min_length, max_length); for(size_t len = min_length; len < max_length; len++) { diff --git a/include/busclique/find_clique.hpp b/include/busclique/find_clique.hpp index d35d0448..e11bb2e5 100644 --- a/include/busclique/find_clique.hpp +++ b/include/busclique/find_clique.hpp @@ -98,7 +98,7 @@ bool find_clique_nice(const cell_cache &cells, } if(minw > maxw) return false; } else { - maxw = (max_length)*6; + maxw = min(cells.topo.dim[0], (max_length)*6); } //we've already found an embedding; now try to find one with shorter chains for(size_t w = minw; w <= maxw; w++) { diff --git a/include/find_embedding/util.hpp b/include/find_embedding/util.hpp index 3e55c479..d40e57e9 100644 --- a/include/find_embedding/util.hpp +++ b/include/find_embedding/util.hpp @@ -184,11 +184,20 @@ class optional_parameters { localInteractionPtr->displayOutput(loglevel, buffer); } + void print_out(int loglevel, const char* format) const { + localInteractionPtr->displayOutput(loglevel, format); + } + + template void print_err(int loglevel, const char* format, Args... args) const { char buffer[1024]; snprintf(buffer, 1024, format, args...); - localInteractionPtr->displayError(loglevel, buffer); + localInteractionPtr->displayError(loglevel, buffer);\ + } + + void print_err(int loglevel, const char* format) const { + localInteractionPtr->displayError(loglevel, format); } template diff --git a/minorminer/busclique.pyx b/minorminer/busclique.pyx index 7db33fb4..df86d576 100644 --- a/minorminer/busclique.pyx +++ b/minorminer/busclique.pyx @@ -23,7 +23,7 @@ import networkx as nx, dwave_networkx as dnx #increment this version any time there is a change made to the cache format, #when yield-improving changes are made to clique algorithms, or when bugs are #fixed in the same. -cdef int __cache_version = 3 +cdef int __cache_version = 4 cdef int __lru_size = 100 cdef dict __global_locks = {'clique': threading.Lock(), @@ -88,7 +88,7 @@ def find_clique_embedding(nodes, g, use_cache = True): 'chimera': _chimera_busgraph}[family] except (AttributeError, KeyError): raise ValueError("g must be either a dwave_networkx.chimera_graph or" - "a dwave_networkx.pegasus_graph") + " a dwave_networkx.pegasus_graph") if use_cache: return busgraph_cache(g).find_clique_embedding(nodes) else: @@ -158,12 +158,18 @@ class busgraph_cache: if rootdir.exists(): dirstack.append(rootdir) while dirstack: - top = dirstack.pop() + top = dirstack[-1] + substack = [] for item in top.iterdir(): if item.is_dir(): - dirstack.append(item) + substack.append(item) else: item.unlink() + if substack: + dirstack.extend(substack) + else: + top.rmdir() + dirstack.pop() def _fetch_cache(self, dirname, compute): """ diff --git a/minorminer/tests/test_busclique.py b/minorminer/tests/test_busclique.py new file mode 100644 index 00000000..2adc240b --- /dev/null +++ b/minorminer/tests/test_busclique.py @@ -0,0 +1,267 @@ +from minorminer import busclique +import minorminer as mm, dwave.embedding as dwe +import unittest, random, itertools, dwave_networkx as dnx, networkx as nx, os + +def subgraph_node_yield(g, q): + """ + returns a subgraph of g produced by removing nodes with probability 1-q + """ + h = g.copy() + h.remove_nodes_from([v for v in h if random.random() > q]) + return h + +def subgraph_edge_yield(g, p): + """ + returns a subgraph of g produced by removing edges with probability 1-p. + """ + h = g.copy() + h.remove_edges_from([e for e in h.edges() if random.random() > p]) + return h + +def subgraph_edge_yield_few_bad(g, p, b): + """ + returns a subgraph of g produced by removing b internal edges and + odd/external edges with probability 1-p. + + g must be a chimera graph or a pegasus graph with coordinate labels + """ + if g.graph['family'] == 'pegasus': + def icpl(p, q): + return p[0] != q[0] + elif g.graph['family'] == 'chimera': + def icpl(p, q): + return p[2] != q[2] + else: + raise ValueError("g is neither a pegasus nor a chimera") + + h = g.copy() + iedges, oedges = [], [] + for e in h.edges(): + if icpl(*e): + iedges.append(e) + elif random.random() > p: + oedges.append(e) + + if len(iedges) >= b: + iedges = random.sample(iedges, b) + else: + iedges = [] + + h.remove_edges_from(itertools.chain(iedges, oedges)) + return h + +def relabel(g, node_label, edge_label, *args, **kwargs): + nodes = node_label(g) + edges = edge_label(g.edges()) + + if g.graph['family'] == 'pegasus': + f = dnx.pegasus_graph + elif g.graph['family'] == 'chimera': + f = dnx.chimera_graph + else: + raise ValueError("g is neither a pegasus nor a chimera") + + return f(*args, edge_list = edges, node_list = nodes, **kwargs) + +def max_chainlength(emb): + if emb: + return max(len(c) for c in emb.values()) + +class TestBusclique(unittest.TestCase): + def __init__(self, *args, **kwargs): + super(TestBusclique, self).__init__(*args, **kwargs) + + self.c16 = dnx.chimera_graph(16) + self.p16 = dnx.pegasus_graph(16) + self.c16c = dnx.chimera_graph(16, coordinates=True, data = False) + self.c428 = dnx.chimera_graph(4, n = 2, t = 8) + self.c248 = dnx.chimera_graph(2, n = 4, t = 8) + self.c42A = dnx.chimera_graph(4, n = 2, t = 9) + c4c_0 = subgraph_node_yield(dnx.chimera_graph(4, coordinates = True), .95) + p4c_0 = subgraph_node_yield(dnx.pegasus_graph(4, coordinates = True), .95) + c4c = [c4c_0, + subgraph_edge_yield(c4c_0, .95), + subgraph_edge_yield_few_bad(c4c_0, .95, 6)] + p4c = [p4c_0, + subgraph_edge_yield(p4c_0, .95), + subgraph_edge_yield_few_bad(p4c_0, .95, 6)] + + p4coords = dnx.pegasus_coordinates(4) + c4coords = dnx.chimera_coordinates(4, 4, 4) + + c4 = [relabel(c, + c4coords.iter_chimera_to_linear, + c4coords.iter_chimera_to_linear_pairs, + 4) for c in c4c] + + p4 = [relabel(p, + p4coords.iter_pegasus_to_linear, + p4coords.iter_pegasus_to_linear_pairs, + 4) for p in p4c] + + p4n = [relabel(p, + p4coords.iter_pegasus_to_nice, + p4coords.iter_pegasus_to_nice_pairs, + 4, nice_coordinates = True) for p in p4c] + + self.c4, self.c4_nd, self.c4_d = list(zip(c4, c4c)) + self.p4, self.p4_nd, self.p4_d = list(zip(p4, p4c, p4n)) + + def test_p16(self): + def reconstruct(nodes): + return dnx.pegasus_graph(16, node_list = nodes) + self.run_battery('p16', self.p16, reconstruct, 180, 17, 172, 15) + + def test_c16(self): + def reconstruct(nodes): + return dnx.chimera_graph(16, node_list = nodes) + self.run_battery('c16', self.c16, reconstruct, 64, 17, 64, 16) + + def run_battery(self, name, g, reconstruct, + cliquesize, cliquelength, + bicliquesize, bicliquelength, + test_python = False, test_nocache = True): + labels = g.graph['labels'] + + test = self.clique_battery(g, reconstruct, + test_python = test_python, + test_nocache = test_nocache) + size, cl = next(test) + self.assertEqual(size, cliquesize) + self.assertEqual(cl, cliquelength) + s = nx.complete_graph(size) + for i, (h, emb, kind, check_cl) in enumerate(test): + print(name, labels, kind) + if check_cl: + self.assertEqual(max_chainlength(emb), cl) + dwe.verify_embedding(emb, s, h) + + test = self.biclique_battery(g, reconstruct) + size, cl = next(test) + self.assertEqual(size, bicliquesize) + if bicliquelength is not None: + self.assertEqual(cl, bicliquelength) + s = nx.complete_bipartite_graph(size, size) + for i, (h, emb, kind) in enumerate(test): + print(name, labels, kind) + if bicliquelength is not None: + self.assertEqual(max_chainlength(emb), bicliquelength) + dwe.verify_embedding(emb, s, h) + + + def clique_battery(self, g, reconstruct, + test_python = False, test_nocache = True): + bgcg = busclique.busgraph_cache(g) + emb0 = bgcg.largest_clique() + size = len(emb0) + cl = max_chainlength(emb0) + N = range(size) + yield size, cl + yield g, emb0, 'g:bcg.lc', True + yield g, bgcg.find_clique_embedding(size), 'g:bcg.fce', True + yield g, busclique.find_clique_embedding(size, g), 'g:bc.fce', True + if test_nocache: + yield (g, + busclique.find_clique_embedding(size, g, use_cache = False), + 'g:bc.fce,nc', True) + yield g, bgcg.largest_clique_by_chainlength(cl), 'g:bc.lcbc', True + if test_python: + if g.graph['family'] == 'chimera': + if g.graph['labels'] == 'int': + # this fails on coordinate-labeled graphs... TODO? + args = size, g.graph['rows'] + kwargs = dict(target_edges = g.edges) + yield (g, + dwe.chimera.find_clique_embedding(*args, **kwargs), + 'g:dwe.fce', True) + if g.graph['family'] == 'pegasus': + kwargs = dict(target_graph = g) + yield (g, dwe.pegasus.find_clique_embedding(size, **kwargs), + 'g:dwe.fce', False) + + nodes = set(itertools.chain.from_iterable(emb0.values())) + h = reconstruct(nodes) + bgch = busclique.busgraph_cache(h) + yield h, busclique.find_clique_embedding(N, h), 'h:bc.fce', True + if test_nocache: + yield (h, busclique.find_clique_embedding(N, h, use_cache = False), + 'h:bc.fce,nc', True) + yield h, bgch.largest_clique(), 'h:bgc.lc', True + yield h, bgch.find_clique_embedding(N), 'h:bgc.fce', True + yield h, bgch.largest_clique_by_chainlength(cl), 'h:bgc.lcbc', True + if test_python: + if g.graph['family'] == 'chimera': + if g.graph['labels'] == 'int': + # this fails on coordinate-labeled graphs... TODO? + args = size, h.graph['rows'] + kwargs = dict(target_edges = h.edges) + yield (h, + dwe.chimera.find_clique_embedding(*args, **kwargs), + 'h:dwe.fce', True) + if g.graph['family'] == 'pegasus': + kwargs = dict(target_graph = h) + yield (h, dwe.pegasus.find_clique_embedding(size, **kwargs), + 'h:dwe.fce', False) + + + def biclique_battery(self, g, reconstruct): + bgcg = busclique.busgraph_cache(g) + emb0 = bgcg.largest_balanced_biclique() + size = len(emb0)//2 + cl = max_chainlength(emb0) + N = range(size) + yield size, cl + yield g, emb0, 'bgc.lbb' + yield (g, bgcg.find_biclique_embedding(N, range(size, 2*size)), + 'bgc.fbe,list') + yield g, bgcg.find_biclique_embedding(size, size), 'bgc.fbe,ints' + + @classmethod + def tearDownClass(cls): + rootdir = busclique.busgraph_cache.cache_rootdir() + if not os.path.exists(rootdir): + raise RuntimeError("cache rootdir not found before cleanup") + busclique.busgraph_cache.clear_all_caches() + if os.path.exists(rootdir): + raise RuntimeError("cache rootdir exists after cleanup") + + def test_chimera_weird_sizes(self): + self.assertRaises(NotImplementedError, + busclique.busgraph_cache, + self.c42A) + + self.assertRaises(NotImplementedError, + busclique.find_clique_embedding, + 999, self.c42A) + + self.assertRaises(NotImplementedError, + busclique.find_clique_embedding, + 999, self.c42A, use_cache = False) + + def reconstructor(m, n, t): + return lambda nodes: dnx.chimera_graph(m, n = n, t = t, + node_list = nodes) + for g, params in (self.c428, (4, 2, 8)), (self.c248, (2, 4, 8)): + reconstruct = reconstructor(*params) + self.run_battery('c%d%d%d'%params, g, reconstruct, 16, 3, 16, None) + + def test_labelings(self): + def reconstructor(g): + return lambda nodes: g.subgraph(nodes).copy() + + names = 'c4_nd', 'c4', 'c4_d', 'p4_nd', 'p4', 'p4_d' + nocache = False, True, True, False, True, True + topos = self.c4_nd, self.c4, self.c4_d, self.p4_nd, self.p4, self.p4_d + + for (name, test_nocache, G) in zip(names, nocache, topos): + g0 = G[0] + bgc = busclique.busgraph_cache(g0) + K = bgc.largest_clique() + B = bgc.largest_balanced_biclique() + for g in G: + self.run_battery(name, g, reconstructor(g), + len(K), max_chainlength(K), + len(B)//2, None, + test_python = test_nocache, + test_nocache = test_nocache) + diff --git a/tests/requirements.txt b/tests/requirements.txt index 07529020..6ba0ccf3 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -5,5 +5,5 @@ homebase==1.0.1 networkx==2.4 numpy==1.18.5 scipy==1.4.1 - +dwave-system nose>=1.3