#!/usr/bin/perl

#
#	TestCluster:  Test an HA cluster
#
#	We only implement one particular kind of testing
#	right now.  But, we expect to add others.
#
#	We currently test for simultaneous startup race
#	conditions.
#

@hosts = ("amymarie", "kathyamy");

$heartbeat="/etc/rc.d/init.d/heartbeat";
$hblib="/usr/lib/heartbeat/heartbeat";
$heartbeatdebug="$hblib -d -d -d -d -d";
$ClusterDebug = 1;
$ClusterDebug = 0;

if ($ClusterDebug) {
	$StartCluster="$heartbeatdebug"
}else{
	$StartCluster="$heartbeat start";
}

#
#	Take a short nap...
#
sub napms {
	local($ms) = @_;
	select(undef, undef, undef, $ms / 1000.0);
}


#
#	Run a command on a remote host...
#
sub runcmd {
	local($host, $cmd) = @_;
	system("/usr/bin/ssh $host '$cmd'");
}

#
#	Start up HA services...
#
sub StartHA {
	local($host) = @_;

	&runcmd($host, $StartCluster);
}

#
#	Stop HA services...
#
sub StopHA {
	local($host) = @_;
	&runcmd($host, "$heartbeat stop");
}

#
#	Append a string to the HA log
#
sub AppendLog {
	local ($host, $message) = @_;
	&runcmd($host, "echo \'$message\' >> /var/log/ha-log");
}

#
#	Append a string to everybody's HA logs
#
sub AppendLogs {
	local($msg) = @_;
	for $host (@hosts) {
		&AppendLog($host, $msg);
	}
}


#
#	Start HA on all nodes, with the specified time skews...
#
#	The idea is that simultaneous startup can be a problem
#	and that there might be time windows in which we have
#	difficulties.
#
#	We are given our time skew numbers in a vector in our
#	input, and we start everything according to this vector
#	of timing offsets.  Hopefully, our caller has some
#	idea of what kinds of starting skews might cause the
#	cluster to have a problem.
#
sub SyncStart {
	local(@delays) = @_;
	local(@pids, $j);
	for $host (@hosts) {
		&StopHA($host);
	}

	sleep(1);
	
	#
	#	Fork off and start HA at the right time
	#
	for ($j=0; $j <= $#hosts; ++$j) {
		if ($pid = fork) {
			# parent process
			push(@pids, $pid);
		}elsif (defined($pid)) {
			# Sleep a little, then start...
			&napms($delays[$j]);
			&StartHA($hosts[$j]);
			exit(0);
		}else{
			print STDERR "Cannot fork\n";
		}
	}
	#
	# Wait for our processes to finish.
	#
	for $pid (@pids) {
		waitpid ($pid, $flags);
	}
}

$> == 0 || die "Must be root [not $>] to run $0";


$EQ="=============";

$min=5000; $max=15000; $incr=250; $repeats = 10;

for ($time=$min; $time <= $max; $time += $incr) {

	print STDERR "\n$EQ $time milliseconds Delay $EQ\n";
	for ($try=1; $try <= $repeats; ++$try) {
		print STDERR "+++Try $try: $time ms delay.\n";
		&AppendLogs(
		"$EQ Try $try: $time ms delay $EQ");
		&SyncStart(0, $time);
		sleep(20);
	}
	print STDERR "\n$EQ $time milliseconds Advance  $EQ\n";
	for ($try=1; $try <= $repeats; ++$try) {
		print STDERR "+++Try $try: $time ms advance.\n";
		&AppendLogs(
		"$EQ Try $try: $time ms advance $EQ");
		&SyncStart($time, 0);
		sleep(20);
	}
}
