Commit e11e46b4 authored by Ranadeep Biswas's avatar Ranadeep Biswas
Browse files

whole lot of changes

parent 81abbf9b
......@@ -15,6 +15,11 @@ serde_json = "*"
serde_yaml = "*"
byteorder = "*"
petgraph = "*"
rayon = "*"
slog = "*"
slog-term = "*"
slog-stream = "*"
chrono = { version = "*", features = ["serde"] }
antidotedb = { path = "../antidotedb" }
[profile.release]
......
......@@ -3,8 +3,10 @@ extern crate byteorder;
extern crate clap;
extern crate dbcop;
use dbcop::db::cluster::{Cluster, ClusterNode, Node, TestParams};
use dbcop::db::history::Transaction;
use std::path::Path;
use dbcop::db::cluster::{Cluster, ClusterNode, Node};
use dbcop::db::history::{HistParams, Transaction};
use clap::{App, Arg};
......@@ -119,6 +121,35 @@ impl AntidoteCluster {
println!("COMMIT ERROR while init");
}
}
self.0.iter_mut().for_each(|x| {
let mut conn = AntidoteDB::connect_with_string(&x.addr);
let timestamp = x.timestamp.clone();
// println!("{:?}", timestamp);
let db_transaction = conn.start_transaction(timestamp.as_ref());
let objs: Vec<_> = (0..n_variable)
.map(|variable| LWWREG::new(&format!("{}", variable), "dbcop"))
.collect();
match conn.mult_read_in_transaction(&objs, &db_transaction) {
Ok(values) => assert!((0..n_variable).all(|var| {
let bytes = values[var].get_reg().get_value();
Cursor::new(bytes).read_u64::<BigEndian>().unwrap() == 0
})),
Err(_) => unreachable!(),
}
match conn.commit_transaction(&db_transaction) {
Ok(commit_time) => {}
Err(_e) => unreachable!(),
}
});
println!("zero init is done");
}
fn drop_database(&self) {}
......@@ -141,38 +172,36 @@ impl Cluster<AntidoteNode> for AntidoteCluster {
fn get_cluster_node(&self, id: usize) -> AntidoteNode {
self.0[id].clone()
}
fn setup_test(&mut self, p: &TestParams) {
self.create_variables(p.n_variable);
fn setup_test(&mut self, p: &HistParams) {
self.create_variables(p.get_n_variable());
}
fn cleanup(&self) {
self.drop_database();
}
fn info(&self) -> String {
"AntidoteDB".to_string()
}
}
fn main() {
let matches = App::new("Antidote")
.version("1.0")
.author("Ranadeep")
.about("verifies a Antidote cluster")
.arg(
Arg::with_name("n_variable")
.long("nval")
.short("v")
.default_value("5"),
)
.about("executes histories on AntidoteDB")
.arg(
Arg::with_name("n_transaction")
.long("ntxn")
.short("t")
.default_value("5"),
Arg::with_name("hist_dir")
.long("dir")
.short("d")
.takes_value(true)
.required(true),
)
.arg(
Arg::with_name("n_event")
.long("nevt")
.short("e")
.default_value("2"),
Arg::with_name("hist_out")
.long("out")
.short("o")
.takes_value(true)
.required(true),
)
.arg(Arg::with_name("history_output").long("output").short("o"))
.arg(
Arg::with_name("ips")
.help("Cluster ips")
......@@ -180,23 +209,15 @@ fn main() {
.required(true),
)
.get_matches();
let hist_dir = Path::new(matches.value_of("hist_dir").unwrap());
let hist_out = Path::new(matches.value_of("hist_out").unwrap());
let ips: Vec<_> = matches.values_of("ips").unwrap().collect();
let mut cluster = AntidoteCluster::new(&ips);
// println!("{:?}", cluster);
cluster.setup();
// test_id, n_variable, n_transaction, n_event
let params = TestParams {
n_variable: matches.value_of("n_variable").unwrap().parse().unwrap(),
n_transaction: matches.value_of("n_transaction").unwrap().parse().unwrap(),
n_event: matches.value_of("n_event").unwrap().parse().unwrap(),
..Default::default()
};
println!("{:?}", params);
cluster.test(&params);
cluster.execute_all(hist_dir, hist_out);
}
......@@ -2,8 +2,10 @@ extern crate clap;
extern crate dbcop;
extern crate postgres;
use dbcop::db::cluster::{Cluster, ClusterNode, Node, TestParams};
use dbcop::db::history::Transaction;
use std::path::Path;
use dbcop::db::cluster::{Cluster, ClusterNode, Node};
use dbcop::db::history::{HistParams, Transaction};
use clap::{App, Arg};
......@@ -158,36 +160,35 @@ impl Cluster<CockroachNode> for CockroachCluster {
fn get_cluster_node(&self, id: usize) -> CockroachNode {
From::from(self.get_node(id))
}
fn setup_test(&mut self, p: &TestParams) {
self.create_variables(p.n_variable);
fn setup_test(&mut self, p: &HistParams) {
self.create_variables(p.get_n_variable());
}
fn cleanup(&self) {
self.drop_database();
}
fn info(&self) -> String {
"CockroachDB".to_string()
}
}
fn main() {
let matches = App::new("Cockroach")
let matches = App::new("CockroachDB")
.version("1.0")
.author("Ranadeep")
.about("verifies a Cockroach cluster")
.arg(
Arg::with_name("n_variable")
.long("nval")
.short("v")
.default_value("5"),
)
.about("executes histories on CockroachDB")
.arg(
Arg::with_name("n_transaction")
.long("ntxn")
.short("t")
.default_value("5"),
Arg::with_name("hist_dir")
.long("dir")
.short("d")
.takes_value(true)
.required(true),
)
.arg(
Arg::with_name("n_event")
.long("nevt")
.short("e")
.default_value("2"),
Arg::with_name("hist_out")
.long("out")
.short("o")
.takes_value(true)
.required(true),
)
.arg(
Arg::with_name("ips")
......@@ -196,23 +197,15 @@ fn main() {
.required(true),
)
.get_matches();
let hist_dir = Path::new(matches.value_of("hist_dir").unwrap());
let hist_out = Path::new(matches.value_of("hist_out").unwrap());
let ips: Vec<_> = matches.values_of("ips").unwrap().collect();
let mut cluster = CockroachCluster::new(&ips);
// println!("{:?}", cluster);
cluster.setup();
// test_id, n_variable, n_transaction, n_event
let params = TestParams {
id: 0,
n_variable: matches.value_of("n_variable").unwrap().parse().unwrap(),
n_transaction: matches.value_of("n_transaction").unwrap().parse().unwrap(),
n_event: matches.value_of("n_event").unwrap().parse().unwrap(),
};
println!("{:?}", params);
cluster.test(&params);
cluster.execute_all(hist_dir, hist_out);
}
......@@ -2,8 +2,10 @@ extern crate clap;
extern crate dbcop;
extern crate mysql;
use dbcop::db::cluster::{Cluster, ClusterNode, Node, TestParams};
use dbcop::db::history::Transaction;
use std::path::Path;
use dbcop::db::cluster::{Cluster, ClusterNode, Node};
use dbcop::db::history::{HistParams, Transaction};
use clap::{App, Arg};
......@@ -156,57 +158,52 @@ impl Cluster<GaleraNode> for GaleraCluster {
fn get_cluster_node(&self, id: usize) -> GaleraNode {
From::from(self.get_node(id))
}
fn setup_test(&self, p: &TestParams) {
self.create_variables(p.n_variable);
fn setup_test(&mut self, p: &HistParams) {
self.create_variables(p.get_n_variable());
}
fn cleanup(&self) {
self.drop_database();
}
fn info(&self) -> String {
"Galera".to_string()
}
}
fn main() {
let matches = App::new("galera")
let matches = App::new("Galera")
.version("1.0")
.author("Ranadeep")
.about("verifies a galera cluster")
.about("executes histories on Galera")
.arg(
Arg::with_name("hist_dir")
.long("dir")
.short("d")
.takes_value(true)
.required(true),
)
.arg(
Arg::with_name("hist_out")
.long("out")
.short("o")
.takes_value(true)
.required(true),
)
.arg(
Arg::with_name("n_variable")
.long("nval")
.short("v")
.default_value("5"),
).arg(
Arg::with_name("n_transaction")
.long("ntxn")
.short("t")
.default_value("5"),
).arg(
Arg::with_name("n_event")
.long("nevt")
.short("e")
.default_value("2"),
).arg(
Arg::with_name("ips")
.help("Cluster ips")
.multiple(true)
.required(true),
).get_matches();
let ips: Vec<_> = matches.values_of("ips").unwrap().collect();
let cluster = GaleraCluster::new(&ips);
)
.get_matches();
// println!("{:?}", cluster);
let hist_dir = Path::new(matches.value_of("hist_dir").unwrap());
let hist_out = Path::new(matches.value_of("hist_out").unwrap());
cluster.setup();
let ips: Vec<_> = matches.values_of("ips").unwrap().collect();
// test_id, n_variable, n_transaction, n_event
let params = TestParams {
id: 0,
n_variable: matches.value_of("n_variable").unwrap().parse().unwrap(),
n_transaction: matches.value_of("n_transaction").unwrap().parse().unwrap(),
n_event: matches.value_of("n_event").unwrap().parse().unwrap(),
};
let mut cluster = GaleraCluster::new(&ips);
println!("{:?}", params);
cluster.setup();
cluster.test(&params);
cluster.execute_all(hist_dir, hist_out);
}
......@@ -15,18 +15,24 @@ args = parser.parse_args()
step = 1
tups = itertools.product(range(5, args.max_var + 1, step),
range(14, args.max_txn + 1, step),
range(14, args.max_evt + 1, step))
range(5, args.max_txn + 1, step),
range(5, args.max_evt + 1, step))
tups = sorted(tups, key=lambda x: sum(x))
# index = tups.index((5, 8, 10))
# index = tups.index((5, 8, 15))
index = tups.index((5, 10, 15))
tups = tups[index:]
# 5 var, 3 client, 4 txn, 3 event
for n_var, n_txn, n_evt in tups:
f_path = "{:03}_{:03}_{:03}_{}".format(
n_var, n_txn, n_evt, uuid.uuid4().hex[:5])
os.mkdir(f_path)
for i in range(2000):
for i in range(250):
with open(os.path.join(f_path, "{:06}.txt".format(i)), "w") as f:
# process = subprocess.Popen("../target/release/examples/cockroachdb -v{} -t{} -e{} 172.19.0.2 172.19.0.3 172.19.0.4".format(
# process = subprocess.Popen("../target/release/examples/galera -v{} -t{} -e{} 172.19.0.4 172.19.0.6 172.19.0.3 172.19.0.7 172.19.0.5".format(
......
......@@ -5,29 +5,34 @@ use consistency::ser::Chains;
use consistency::util::EdgeClosure;
use petgraph::algo::astar;
use petgraph::dot::{Config, Dot};
// use petgraph::dot::{Config, Dot};
use petgraph::graph::node_index;
use petgraph::Graph;
use slog::Logger;
pub struct Causal {
chains: Chains,
co_closure: EdgeClosure,
pub vis_PG: Graph<usize, usize>,
pub co_PG: Graph<usize, usize>,
pub vis_pg: Graph<usize, usize>,
pub co_pg: Graph<usize, usize>,
log: Logger,
}
impl Causal {
pub fn new(
n_sizes: &Vec<usize>,
txns_info: &HashMap<(usize, usize), (HashMap<usize, (usize, usize)>, HashSet<usize>)>,
log: Logger,
) -> Self {
let mut chains = Chains::new(&n_sizes, &txns_info);
let mut chains = Chains::new(&n_sizes, &txns_info, log.clone());
chains.preprocess_wr();
Causal {
chains: chains,
co_closure: EdgeClosure::new(),
vis_PG: Graph::new(),
co_PG: Graph::new(),
vis_pg: Graph::new(),
co_pg: Graph::new(),
log,
}
}
......@@ -35,28 +40,28 @@ impl Causal {
for po in self.chains.tuple_to_id.iter().skip(1) {
for &id in po.iter().rev().skip(1) {
if self.chains.vis_closure.contains(id + 1, id) {
println!("found cycle in WR");
info!(self.log, "found cycle in WR");
return false;
}
self.chains.vis_closure.add_edge(id, id + 1);
self.co_closure.add_edge(id, id + 1);
self.vis_PG
self.vis_pg
.extend_with_edges(&[(id as u32, (id + 1) as u32, 1)]);
self.co_PG
self.co_pg
.extend_with_edges(&[(id as u32, (id + 1) as u32, 1)]);
}
if let Some(&u) = po.first() {
if self.chains.vis_closure.contains(u, self.chains.root_txn_id) {
println!("found cycle in WR");
info!(self.log, "found cycle in WR");
return false;
}
self.chains.vis_closure.add_edge(self.chains.root_txn_id, u);
self.co_closure.add_edge(self.chains.root_txn_id, u);
self.vis_PG
self.vis_pg
.extend_with_edges(&[(self.chains.root_txn_id as u32, u as u32, 1)]);
self.co_PG
self.co_pg
.extend_with_edges(&[(self.chains.root_txn_id as u32, u as u32, 1)]);
}
}
......@@ -65,14 +70,14 @@ impl Causal {
for (&u, vs) in info {
for &v in vs.iter() {
if self.chains.vis_closure.contains(v, u) {
println!("found cycle in WR");
info!(self.log, "found cycle in WR");
return false;
}
self.chains.vis_closure.add_edge(u, v);
self.co_closure.add_edge(u, v);
self.vis_PG.extend_with_edges(&[(u as u32, v as u32, 1)]);
self.co_PG.extend_with_edges(&[(u as u32, v as u32, 1)]);
self.vis_pg.extend_with_edges(&[(u as u32, v as u32, 1)]);
self.co_pg.extend_with_edges(&[(u as u32, v as u32, 1)]);
}
}
}
......@@ -92,13 +97,13 @@ impl Causal {
for (&u_, _) in wr_x.iter() {
if u != u_ && v != u_ {
// if self.chains.vis_closure.contains(u, u_) {
// println!(
// info!(self.log,
// "adding RW ({1}, {2}), WR_{3}({0}, {1}), {3} in W({2}), VIS({0}, {2})",
// u, v, u_, _x
// );
// if self.chains.vis_closure.contains(u_, v) {
// // println!("cycle: {0} -> {1} -> {0}", v, u_);
// println!(
// // info!(self.log,"cycle: {0} -> {1} -> {0}", v, u_);
// info!(self.log,
// "VIS*-RW cycle: {0:?} -> {1:?} -> {0:?}",
// self.id_to_tuple[v], self.id_to_tuple[u_]
// );
......@@ -107,16 +112,18 @@ impl Causal {
// new_rw_edge.push((v, u_));
// }
if self.chains.vis_closure.contains(u_, v) {
// println!(
// info!(self.log,
// "adding WW ({2}, {0}), WR_{3}({0}, {1}), {3} in W({2}), VIS({2}, {1})",
// u, v, u_, _x
// );
if self.chains.vis_closure.contains(u, u_) {
// println!("cycle: {0} -> {1} -> {0}", u_, u);
println!(
// info!(self.log,"cycle: {0} -> {1} -> {0}", u_, u);
info!(
self.log,
"cycle: {0:?} co {1:?} vis {0:?}",
self.chains.id_to_tuple[u_], self.chains.id_to_tuple[u]
self.chains.id_to_tuple[u_],
self.chains.id_to_tuple[u]
);
return false;
}
......@@ -133,32 +140,36 @@ impl Causal {
for (u, v) in new_ww_edge {
if self.co_closure.contains(v, u) {
// println!("cycle: {0} -> {1} -> {0}", u, v);
println!(
// info!(self.log,"cycle: {0} -> {1} -> {0}", u, v);
info!(
self.log,
"cycle: {0:?} co {1:?} co {0:?}",
self.chains.id_to_tuple[u], self.chains.id_to_tuple[v]
self.chains.id_to_tuple[u],
self.chains.id_to_tuple[v]
);
let co_path = astar(
&self.co_PG,
&self.co_pg,
node_index(v),
|finish| finish == node_index(u),
|e| *e.weight(),
|_| 0,
);
println!("{:?}", co_path);
info!(self.log, "{:?}", co_path);
if let Some((_, ref path)) = co_path {
for win2 in path.windows(2) {
let p_u = win2[0];
let p_v = win2[1];
if self.vis_PG.contains_edge(p_u, p_v) {
println!(
if self.vis_pg.contains_edge(p_u, p_v) {
info!(
self.log,
"so/wr, {:?} {:?}",
self.chains.id_to_tuple[p_u.index()],
self.chains.id_to_tuple[p_v.index()]
);
} else {
println!(
info!(
self.log,
"co, {:?}, {:?}",
self.chains.id_to_tuple[p_u.index()],
self.chains.id_to_tuple[p_v.index()]
......@@ -167,7 +178,7 @@ impl Causal {
let &(u, v, u_, _x) =
ww_reason.get(&(p_u.index(), p_v.index())).unwrap();
let vis_path = astar(
&self.co_PG,
&self.co_pg,
node_index(u_),
|finish| finish == node_index(v),
|e| *e.weight(),
......@@ -175,32 +186,41 @@ impl Causal {
)
.unwrap()
.1;
println!("reason:");
println!(
info!(self.log, "reason:");
info!(
self.log,
"vis path {:?} -> {:?}",
self.chains.id_to_tuple[u_], self.chains.id_to_tuple[v]
self.chains.id_to_tuple[u_],
self.chains.id_to_tuple[v]
);
let mut path_vec = Vec::new();
for e in vis_path {
print!("{:?} -> ", self.chains.id_to_tuple[e.index()]);
path_vec.push(format!(
"{:?}",
self.chains.id_to_tuple[e.index()]
));
}
println!();
println!(
info!(self.log, "{:?}", path_vec);
info!(
self.log,
"{:?} wr_{:?} {:?}",
self.chains.id_to_tuple[u], _x, self.chains.id_to_tuple[v]
self.chains.id_to_tuple[u],
_x,
self.chains.id_to_tuple[v]
);
}
}
}
}
println!(
"co, {:?}, {:?}",
self.chains.id_to_tuple[u], self.chains.id_to_tuple[v]
info!(
self.log,
"co, {:?}, {:?}", self.chains.id_to_tuple[u], self.chains.id_to_tuple[v]
);
{
let &(u, v, u_, _x) = ww_reason.get(&(u, v)).unwrap();
let vis_path = astar(
&self.co_PG,