/*
 * Dijjer - A Peer to Peer HTTP Cache
 * Copyright (C) 2004,2005 Change.Tv, Inc
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
package freenet.io.xfer;

import static java.util.concurrent.TimeUnit.SECONDS;

import freenet.io.comm.AsyncMessageFilterCallback;
import freenet.io.comm.ByteCounter;
import freenet.io.comm.DMT;
import freenet.io.comm.DisconnectedException;
import freenet.io.comm.Message;
import freenet.io.comm.MessageCore;
import freenet.io.comm.MessageFilter;
import freenet.io.comm.NotConnectedException;
import freenet.io.comm.PeerContext;
import freenet.io.comm.RetrievalException;
import freenet.io.comm.SlowAsyncMessageFilterCallback;
import freenet.node.PeerNode;
import freenet.node.SyncSendWaitedTooLongException;
import freenet.support.BitArray;
import freenet.support.Buffer;
import freenet.support.LogThresholdCallback;
import freenet.support.Logger;
import freenet.support.Logger.LogLevel;
import freenet.support.Ticker;
import freenet.support.TimeUtil;
import freenet.support.io.NativeThread;
import freenet.support.math.MedianMeanRunningAverage;

/**
 * IMPORTANT: The receiver can cancel the incoming transfer. This may or may not, 
 * depending on the caller, result in the PRB being cancelled, and thus propagate back to
 * the originator.
 * 
 * This allows for a weak DoS, in that a node can start a request and then cancel it, 
 * having wasted a certain amount of upstream bandwidth on transferring data, especially
 * if upstream has lots of bandwidth and the attacker has limited bandwidth in the victim
 * -> attacker direction. However this behaviour can be detected fairly easily.
 * 
 * If we allow receiver cancels and don't propagate, a more serious DoS is possible. If we
 * don't allow receiver cancels, we have to get rid of turtles, and massively tighten up
 * transfer timeouts.
 * 
 * However, if we do that, we have to consider that a node might be able to connect, max
 * out the bandwidth with transfers, and then disconnect, avoiding the need to spend
 * bandwidth on receiving all the data; and then reconnect, after it's confident that the
 * transfers to it will have been cancelled. Or not reconnect at all, on opennet - just 
 * use a different identity. Downstream bandwidth is very cheap for small-scale attackers,
 * but if this is a usable force multiplier it could still be a good DoS if we went that
 * way.
 * 
 * But if we did get rid of receiver cancels, it *would* mean we could get rid of a lot of
 * code - e.g. the ReceiverAbortHandler, which in some cases (e.g RequestHandler) is 
 * complex and involves complex security tradeoffs. It would also make transfers 
 * significantly more reliable.
 * 
 * @author ian
 */
public class BlockReceiver implements AsyncMessageFilterCallback {

	private static volatile boolean logMINOR;

	static {
		Logger.registerLogThresholdCallback(new LogThresholdCallback(){
			@Override
			public void shouldUpdate(){
				logMINOR = Logger.shouldLog(LogLevel.MINOR, this);
			}
		});
	}

	public interface BlockReceiverTimeoutHandler {
		
		/** After a block times out, we call this callback. Once it returns, we cancel the
		 * PRB and wait for a cancel message or the second timeout. Hence, if the problem
		 * is on the node sending the data, we will get the first timeout then the second
		 * (fatal) timeout. But if the problem is upstream, we will only get the first
		 * timeout. 
		 * 
		 * Simple requests will need to implement this and transfer ownership of
		 * the request to this node, because the source node will end the request as soon
		 * as it sees the transfer cancel resulting from the PRB being cancelled; 
		 * assigning the UID to ourselves keeps it consistent, and thus avoids severe load
		 * management problems (resulting in e.g. constantly sending requests to a node 
		 * which are then rejected because we think we have capacity when we don't). */
		void onFirstTimeout();
		
		/** After the first timeout, we wait for either a cancel message (sendAborted 
		 * here), or the second timeout. If we get the second timeout, the problem was
		 * caused by the node we are receiving the data from, rather than upstream. In
		 * which case, we may need to take severe action against the node responsible, 
		 * because we do not know whether or not it thinks the transfer is still running.
		 * If it is still running and yet we cancel it, we will think that there is 
		 * capacity for more requests on the node when there isn't, resulting in load 
		 * management problems as above. */
		void onFatalTimeout(PeerContext source);
	}

	/*
	 * RECEIPT_TIMEOUT must be less than 60 seconds because BlockTransmitter times out after not
	 * hearing from us in 60 seconds. Without contact from the transmitter, we will try sending
	 * at most MAX_CONSECUTIVE_MISSING_PACKET_REPORTS every RECEIPT_TIMEOUT to recover.
	 */
	public final long RECEIPT_TIMEOUT;
	public static final long RECEIPT_TIMEOUT_REALTIME = SECONDS.toMillis(10);
	public static final long RECEIPT_TIMEOUT_BULK = SECONDS.toMillis(30);
	// TODO: This should be proportional to the calculated round-trip-time, not a constant
	public final long MAX_ROUND_TRIP_TIME;
	public static final int MAX_CONSECUTIVE_MISSING_PACKET_REPORTS = 4;
	public static final int MAX_SEND_INTERVAL = 500;
	public static final long CLEANUP_TIMEOUT = SECONDS.toMillis(5);
	// After 15 seconds, the receive is overdue and will cause backoff.
	public static final long TOO_LONG_TIMEOUT = SECONDS.toMillis(15);
	/** sendAborted is not sent at the realtime/bulk priority. Most of the two
	 * stage timeout stuff uses 60 seconds, it's a good number. */
	public static final long ACK_TRANSFER_FAILED_TIMEOUT = SECONDS.toMillis(60);
	PartiallyReceivedBlock _prb;
	PeerContext _sender;
	long _uid;
	MessageCore _usm;
	ByteCounter _ctr;
	Ticker _ticker;
	boolean sentAborted;
	private MessageFilter discardFilter;
	private long discardEndTime;
	private boolean senderAborted;
	private final boolean _realTime;
//	private final boolean _doTooLong;
	private final BlockReceiverTimeoutHandler _timeoutHandler;
	private final boolean completeAfterAckedAllReceived;

	/**
	 * @param usm
	 * @param sender
	 * @param uid
	 * @param prb
	 * @param ctr
	 * @param ticker
	 * @param doTooLong
	 * @param realTime
	 * @param timeoutHandler
	 * @param completeAfterAckedAllReceived If true, we need to call completion
	 * only after we have received an ack to the allReceived message. Generally,
	 * handlers want to complete early (=false), so the slot is freed up and can 
	 * be reused by the other side; senders want to complete late, so they don't 
	 * end up reusing the slot before the handler has completed (=true).
	 */
	public BlockReceiver(MessageCore usm, PeerContext sender, long uid, PartiallyReceivedBlock prb, ByteCounter ctr, Ticker ticker, boolean doTooLong, boolean realTime, BlockReceiverTimeoutHandler timeoutHandler, boolean completeAfterAckedAllReceived) {
		BlockReceiverTimeoutHandler nullTimeoutHandler = new BlockReceiverTimeoutHandler() {

			@Override
			public void onFirstTimeout() {
				// Do nothing
			}

			@Override
			public void onFatalTimeout(PeerContext source) {
				// Do nothing
			}

		};
		_timeoutHandler = timeoutHandler == null ? nullTimeoutHandler : timeoutHandler;
		_sender = sender;
		_prb = prb;
		_uid = uid;
		_usm = usm;
		_ctr = ctr;
		_ticker = ticker;
		_realTime = realTime;
		this.completeAfterAckedAllReceived = completeAfterAckedAllReceived;
		RECEIPT_TIMEOUT = _realTime ? RECEIPT_TIMEOUT_REALTIME : RECEIPT_TIMEOUT_BULK;
		MAX_ROUND_TRIP_TIME = RECEIPT_TIMEOUT;
//		_doTooLong = doTooLong;
	}

	private void sendAborted(int reason, String desc) throws NotConnectedException {
		synchronized(this) {
			if(sentAborted) return;
			sentAborted = true;
		}
		_usm.send(_sender, DMT.createSendAborted(_uid, reason, desc), _ctr);
	}
	
	public interface BlockReceiverCompletion {

		public void blockReceived(byte[] buf);

		public void blockReceiveFailed(RetrievalException e);

	}
	
	private BlockReceiverCompletion callback;
	
	private long startTime;
	
	// If false, don't check for duplicate messages from the sender.
	// Turn off if e.g. we know that the PRB is already partially received when we start the transfer.
	// This prevents malicious or broken nodes from trickling transfers forever by sending the same packets over and over.
	static final boolean CHECK_DUPES = true;
	
	private boolean gotAllSent;
	
	private AsyncMessageFilterCallback notificationWaiter = new SlowAsyncMessageFilterCallback() {

		@Override
		public void onMatched(Message m1) {
            if(logMINOR)
            	Logger.minor(this, "Received "+m1);
            if ((m1 != null) && m1.getSpec().equals(DMT.sendAborted)) {
				String desc=m1.getString(DMT.DESCRIPTION);
				if (desc.indexOf("Upstream")<0)
					desc="Upstream transmit error: "+desc;
				_prb.abort(m1.getInt(DMT.REASON), desc, false);
				synchronized(BlockReceiver.this) {
					senderAborted = true;
				}
				complete(m1.getInt(DMT.REASON), desc);
				return;
			}
            boolean truncateTimeout = false;
			if ((m1 != null) && (m1.getSpec().equals(DMT.packetTransmit))) {
				// packetTransmit received
				int packetNo = m1.getInt(DMT.PACKET_NO);
				BitArray sent = (BitArray) m1.getObject(DMT.SENT);
				Buffer data = (Buffer) m1.getObject(DMT.DATA);
				int missing = 0;
				try {
					synchronized(BlockReceiver.this) {
						if(completed) return;
					}
					if(CHECK_DUPES && _prb.isReceived(packetNo)) {
						// Transmitter sent the same packet twice?!?!?
						Logger.error(this, "Already received the packet - DoS??? on "+this+" uid "+_uid+" from "+_sender);
						// Does not extend timeouts.
						truncateTimeout = true;
					} else {
						_prb.addPacket(packetNo, data);
						if(logMINOR) {
							synchronized(BlockReceiver.this) {
								long interval = System.currentTimeMillis() - timeStartedWaiting;
								Logger.minor(this, "Packet interval: "+interval+" = "+TimeUtil.formatTime(interval, 2, true)+" from "+_sender);
							}
						}
						// Check that we have what the sender thinks we have
						for (int x = 0; x < sent.getSize(); x++) {
							if (sent.bitAt(x) && !_prb.isReceived(x)) {
								missing++;
							}
						}
						if(logMINOR && missing != 0) 
							Logger.minor(this, "Packets which the sender says it has sent but we have not received: "+missing);
					}
				} catch (AbortedException e) {
					// We didn't cause it?!
					Logger.error(this, "Caught in receive - probably a bug as receive sets it: "+e, e);
					complete(RetrievalException.UNKNOWN, "Aborted?");
					return;
				}
			} else if (m1 != null && m1.getSpec().equals(DMT.allSent)) {
				synchronized(BlockReceiver.this) {
					if(completed) return;
					if(gotAllSent)
						// Multiple allSent's don't extend the timeouts.
						truncateTimeout = true;
					gotAllSent = true;
				}
			}
			try {
				if(_prb.allReceived()) {
					try {
						Message m = DMT.createAllReceived(_uid);
						if(completeAfterAckedAllReceived) {
							try {
								// FIXME layer violation
								// FIXME make asynchronous
								((PeerNode)_sender).sendSync(m, _ctr, _realTime);
							} catch (SyncSendWaitedTooLongException e) {
								// Complete anyway.
							}
						} else {
							_usm.send(_sender, m, _ctr);
						}
						discardEndTime=System.currentTimeMillis()+CLEANUP_TIMEOUT;
						discardFilter=relevantMessages(CLEANUP_TIMEOUT);
						maybeResetDiscardFilter();
					} catch (NotConnectedException e1) {
						// Ignore, we've got it.
						if(logMINOR) Logger.minor(this, "Got data but can't send allReceived to "+_sender+" as is disconnected");
					}
					long endTime = System.currentTimeMillis();
					long transferTime = (endTime - startTime);
					if(logMINOR) {
						synchronized(avgTimeTaken) {
							avgTimeTaken.report(transferTime);
							Logger.minor(this, "Block transfer took "+transferTime+"ms - average is "+avgTimeTaken);
						}
					}
					complete(_prb.getBlock());
					return;
				}
			} catch (AbortedException e1) {
				// We didn't cause it?!
				Logger.error(this, "Caught in receive - probably a bug as receive sets it: "+e1, e1);
				complete(RetrievalException.UNKNOWN, "Aborted?");
				return;
			}
			try {
				// Even if timeout <= 0, we still add the filter, because we want to receive any messages that are already buffered before we timeout.
				waitNotification(truncateTimeout);
			} catch (DisconnectedException e) {
				onDisconnect(null);
				return;
			}
		}
		
		@Override
		public boolean shouldTimeout() {
			return completed;
		}
		
		@Override
		public void onTimeout() {
			synchronized(this) {
				if(completed) return;
			}
			try {
				if(_prb.allReceived()) return;
				_prb.abort(RetrievalException.SENDER_DIED, "Sender unresponsive to resend requests", false);
				complete(RetrievalException.SENDER_DIED,
						"Sender unresponsive to resend requests");
				
				_timeoutHandler.onFirstTimeout();
				// If upstream caused the problem, then sender will itself timeout
				// and will tell us. So wait for a timeout.
				// It is important for load management that the two sides agree on the number of transfers happening.
				// Therefore we need to not complete until the other side has acknowledged that the transfer has been cancelled.
				MessageFilter mfSendAborted = MessageFilter.create().setTimeout(ACK_TRANSFER_FAILED_TIMEOUT).setType(DMT.sendAborted).setField(DMT.UID, _uid).setSource(_sender);
				try {
					_usm.addAsyncFilter(mfSendAborted, new SlowAsyncMessageFilterCallback() {

						@Override
						public void onMatched(Message m) {
							// Ok.
							if(logMINOR) Logger.minor(this, "Transfer cancel acknowledged");
						}

						@Override
						public boolean shouldTimeout() {
							return false;
						}

						@Override
						public void onTimeout() {
							Logger.error(this, "Other side did not acknowlege transfer failure on "+BlockReceiver.this);
							_timeoutHandler.onFatalTimeout(_sender);
						}

						@Override
						public void onDisconnect(PeerContext ctx) {
							// Ok.
						}

						@Override
						public void onRestarted(PeerContext ctx) {
							// Ok.
						}

						@Override
						public int getPriority() {
							return NativeThread.NORM_PRIORITY;
						}
						
					}, _ctr);
				} catch (DisconnectedException e) {
					// Ignore
				}
				
				return;
			} catch (AbortedException e) {
				// We didn't cause it?!
				Logger.error(this, "Caught in receive - probably a bug as receive sets it: "+e, e);
				complete(RetrievalException.UNKNOWN, "Aborted?");
				return;
			}
		}
		
		@Override
		public void onDisconnect(PeerContext ctx) {
			complete(RetrievalException.SENDER_DISCONNECTED, RetrievalException.getErrString(RetrievalException.SENDER_DISCONNECTED));
		}
		
		@Override
		public void onRestarted(PeerContext ctx) {
			complete(RetrievalException.SENDER_DISCONNECTED, RetrievalException.getErrString(RetrievalException.SENDER_DISCONNECTED));
		}

		@Override
		public int getPriority() {
			return NativeThread.NORM_PRIORITY;
		}
		
	};
	
	private boolean completed;
	
	private void complete(int reason, String description) {
		synchronized(this) {
			if(completed) {
				if(logMINOR) Logger.minor(this, "Already completed");
				return;
			}
			completed = true;
		}
		if(logMINOR)
			Logger.minor(this, "Transfer failed: ("+(_realTime?"realtime":"bulk")+") "+reason+" : "+description+" on "+_uid+" from "+_sender);
		_prb.removeListener(myListener);
		byte[] block = _prb.abort(reason, description, false);
		if(block == null) {
			// Expected behaviour.
			// Send the abort whether we have received one or not.
			// If we are cancelling due to failing to turtle, we need to tell the sender
			// this otherwise he will keep sending, wasting a lot of bandwidth on packets
			// that we will ignore. If we are cancelling because the sender has told us 
			// to, we need to acknowledge that.
			try {
				sendAborted(_prb._abortReason, _prb._abortDescription);
			} catch (NotConnectedException e) {
				// Ignore at this point.
			}
			callback.blockReceiveFailed(new RetrievalException(reason, description));
		} else {
			Logger.error(this, "Succeeded in complete("+reason+","+description+") on "+this, new Exception("error"));
			callback.blockReceived(block);
		}
		decRunningBlockReceives();
	}

	private void complete(byte[] ret) {
		synchronized(this) {
			if(completed) {
				if(logMINOR) Logger.minor(this, "Already completed");
				return;
			}
			completed = true;
		}
		_prb.removeListener(myListener);
		callback.blockReceived(ret);
		decRunningBlockReceives();
	}

	private long timeStartedWaiting = -1;
	
	private void waitNotification(boolean truncateTimeout) throws DisconnectedException {
		long timeout;
		long now = System.currentTimeMillis();
		synchronized(this) {
			if(truncateTimeout) {
				timeout = (int)Math.min(timeStartedWaiting + RECEIPT_TIMEOUT - now, RECEIPT_TIMEOUT);
			} else {
				timeStartedWaiting = now;
				timeout = RECEIPT_TIMEOUT;
			}
		}
		_usm.addAsyncFilter(relevantMessages(timeout), notificationWaiter, _ctr);
	}

	private MessageFilter relevantMessages(long timeout) {
		MessageFilter mfPacketTransmit = MessageFilter.create().setTimeout(timeout).setType(DMT.packetTransmit).setField(DMT.UID, _uid).setSource(_sender);
		MessageFilter mfAllSent = MessageFilter.create().setTimeout(timeout).setType(DMT.allSent).setField(DMT.UID, _uid).setSource(_sender);
		MessageFilter mfSendAborted = MessageFilter.create().setTimeout(timeout).setType(DMT.sendAborted).setField(DMT.UID, _uid).setSource(_sender);
		return mfPacketTransmit.or(mfAllSent.or(mfSendAborted));
	}

	PartiallyReceivedBlock.PacketReceivedListener myListener;
	
	public void receive(BlockReceiverCompletion callback) {
		startTime = System.currentTimeMillis();
		this.callback = callback;
		synchronized(_prb) {
			try {
				_prb.addListener(myListener = new PartiallyReceivedBlock.PacketReceivedListener() {;

					@Override
					public void packetReceived(int packetNo) {
						// Ignore
					}

					@Override
					public void receiveAborted(int reason, String description) {
						complete(reason, description);
					}
				});
			} catch (AbortedException e) {
				try {
					callback.blockReceived(_prb.getBlock());
					return;
				} catch (AbortedException e1) {
					e = e1;
				}
				callback.blockReceiveFailed(new RetrievalException(_prb._abortReason, _prb._abortDescription));
				return;
			}
		}
		incRunningBlockReceives();
		try {
			waitNotification(false);
		} catch (DisconnectedException e) {
			RetrievalException retrievalException = new RetrievalException(RetrievalException.SENDER_DISCONNECTED);
			_prb.abort(retrievalException.getReason(), retrievalException.toString(), true /* kind of, it shouldn't count towards the stats anyway */);
			callback.blockReceiveFailed(retrievalException);
			decRunningBlockReceives();
		} catch(RuntimeException e) {
			decRunningBlockReceives();
			throw e;
		} catch (Error e) {
			decRunningBlockReceives();
			throw e;
		}
	}
	
	private static MedianMeanRunningAverage avgTimeTaken = new MedianMeanRunningAverage();
	
	private void maybeResetDiscardFilter() {
		long timeleft=discardEndTime-System.currentTimeMillis();
		if (timeleft>0) {
			try {
				discardFilter.setTimeout((int)timeleft);
				_usm.addAsyncFilter(discardFilter, this, _ctr);
			} catch (DisconnectedException e) {
				//ignore
			}
		}
	}
	
	/**
	 * Used to discard leftover messages, usually just packetTransmit and allSent.
	 * allSent, is quite common, as the receive() routine usually quits immeadiately on receiving all packets.
	 * packetTransmit is less common, when receive() requested what it thought was a missing packet, only reordered.
	 */
	@Override
	public void onMatched(Message m) {
		if (logMINOR)
			Logger.minor(this, "discarding message post-receive: "+m);
		maybeResetDiscardFilter();												   
	}
	
	@Override
	public boolean shouldTimeout() {
		return false;
	}
	
	@Override
	public void onTimeout() {
		//ignore
	}

	@Override
	public void onDisconnect(PeerContext ctx) {
		// Ignore
	}

	@Override
	public void onRestarted(PeerContext ctx) {
		// Ignore
	}

	public synchronized boolean senderAborted() {
		return senderAborted;
	}
	
	static int runningBlockReceives = 0;
	
	private void incRunningBlockReceives() {
		if(logMINOR) Logger.minor(this, "Starting block receive "+_uid);
		synchronized(BlockReceiver.class) {
			runningBlockReceives++;
			if(logMINOR) Logger.minor(BlockTransmitter.class, "Started a block receive, running: "+runningBlockReceives);
		}
	}
	
	private void decRunningBlockReceives() {
		if(logMINOR) Logger.minor(this, "Stopping block receive "+_uid);
		synchronized(BlockReceiver.class) {
			runningBlockReceives--;
			if(logMINOR) Logger.minor(BlockTransmitter.class, "Finished a block receive, running: "+runningBlockReceives);
		}
	}

	public synchronized static int getRunningReceives() {
		return runningBlockReceives;
	}
	
	@Override
	public String toString() {
		return super.toString()+":"+_uid+":"+_sender.shortToString();
	}
	
}
