/*
 * Created on Jul 8, 2009
 * Created by Paul Gardner
 * 
 * Copyright (C) Azureus Software, Inc, All Rights Reserved.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 */


package com.aelitis.azureus.core.content;

import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.lang.ref.WeakReference;
import java.net.URL;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

import org.gudy.azureus2.core3.config.COConfigurationManager;
import org.gudy.azureus2.core3.config.ParameterListener;
import org.gudy.azureus2.core3.download.DownloadManagerState;
import org.gudy.azureus2.core3.torrent.TOTorrent;
import org.gudy.azureus2.core3.torrent.TOTorrentAnnounceURLGroup;
import org.gudy.azureus2.core3.torrent.TOTorrentAnnounceURLSet;
import org.gudy.azureus2.core3.util.AENetworkClassifier;
import org.gudy.azureus2.core3.util.AERunnable;
import org.gudy.azureus2.core3.util.AESemaphore;
import org.gudy.azureus2.core3.util.AsyncDispatcher;
import org.gudy.azureus2.core3.util.BDecoder;
import org.gudy.azureus2.core3.util.BEncoder;
import org.gudy.azureus2.core3.util.Base32;
import org.gudy.azureus2.core3.util.ByteArrayHashMap;
import org.gudy.azureus2.core3.util.ByteFormatter;
import org.gudy.azureus2.core3.util.Debug;
import org.gudy.azureus2.core3.util.FileUtil;
import org.gudy.azureus2.core3.util.RandomUtils;
import org.gudy.azureus2.core3.util.SHA1Simple;
import org.gudy.azureus2.core3.util.SimpleTimer;
import org.gudy.azureus2.core3.util.StringInterner;
import org.gudy.azureus2.core3.util.SystemTime;
import org.gudy.azureus2.core3.util.TimerEvent;
import org.gudy.azureus2.core3.util.TimerEventPerformer;
import org.gudy.azureus2.core3.util.TorrentUtils;
import org.gudy.azureus2.plugins.PluginInterface;
import org.gudy.azureus2.plugins.PluginListener;
import org.gudy.azureus2.plugins.ddb.DistributedDatabase;
import org.gudy.azureus2.plugins.ddb.DistributedDatabaseTransferType;
import org.gudy.azureus2.plugins.disk.DiskManagerFileInfo;
import org.gudy.azureus2.plugins.download.Download;
import org.gudy.azureus2.plugins.download.DownloadManager;
import org.gudy.azureus2.plugins.download.DownloadManagerListener;
import org.gudy.azureus2.plugins.torrent.Torrent;
import org.gudy.azureus2.plugins.torrent.TorrentAttribute;
import org.gudy.azureus2.plugins.utils.search.SearchException;
import org.gudy.azureus2.plugins.utils.search.SearchInstance;
import org.gudy.azureus2.plugins.utils.search.SearchObserver;
import org.gudy.azureus2.pluginsimpl.local.PluginCoreUtils;
import org.gudy.azureus2.pluginsimpl.local.ddb.DDBaseImpl;

import com.aelitis.azureus.core.AzureusCore;
import com.aelitis.azureus.core.cnetwork.ContentNetwork;
import com.aelitis.azureus.core.proxy.impl.AEPluginProxyHandler;
import com.aelitis.azureus.core.security.CryptoManagerFactory;
import com.aelitis.azureus.core.tag.Tag;
import com.aelitis.azureus.core.tag.TagManager;
import com.aelitis.azureus.core.tag.TagManagerFactory;
import com.aelitis.azureus.core.tag.TagType;
import com.aelitis.azureus.core.torrent.PlatformTorrentUtils;
import com.aelitis.azureus.core.util.CopyOnWriteList;
import com.aelitis.azureus.core.util.FeatureAvailability;
import com.aelitis.azureus.core.util.bloom.BloomFilter;
import com.aelitis.azureus.core.util.bloom.BloomFilterFactory;
import com.aelitis.azureus.plugins.dht.DHTPlugin;
import com.aelitis.azureus.plugins.dht.DHTPluginContact;
import com.aelitis.azureus.plugins.dht.DHTPluginInterface;
import com.aelitis.azureus.plugins.dht.DHTPluginOperationListener;
import com.aelitis.azureus.plugins.dht.DHTPluginValue;
import com.aelitis.azureus.util.ImportExportUtils;

public class 
RelatedContentManager
{
	public static final long FILE_ASSOC_MIN_SIZE	= 50*1024*1024;
	
	public static final int RCM_SEARCH_PROPERTY_CONTENT_NETWORK	= 50000;	// don't change these, used in plugin
	public static final int RCM_SEARCH_PROPERTY_TRACKER_KEYS	= 50001;
	public static final int RCM_SEARCH_PROPERTY_WEB_SEED_KEYS	= 50002;
	public static final int RCM_SEARCH_PROPERTY_TAGS			= 50003;
	public static final int RCM_SEARCH_PROPERTY_NETWORKS		= 50004;
	
	private static final boolean 	TRACE 			= false;
		
	private static final int	MAX_HISTORY					= 16;
	private static final int	MAX_TITLE_LENGTH			= 80;
	private static final int	MAX_CONCURRENT_PUBLISH		= 2;
	
	private static final int	TEMPORARY_SPACE_DELTA	= 50;
	
	private static final int	MAX_RANK	= 100;
	
	private static final String	CONFIG_FILE 				= "rcm.config";
	private static final String	PERSIST_DEL_FILE 			= "rcmx.config";
	
	private static final String	CONFIG_TOTAL_UNREAD	= "rcm.numunread.cache";
	
	private static RelatedContentManager	singleton;
	private static AzureusCore				core;
	
	protected static final int TIMER_PERIOD					= 30*1000;
	
	private static final int CONFIG_SAVE_CHECK_PERIOD		= 60*1000;
	private static final int CONFIG_SAVE_PERIOD				= 5*60*1000;
	private static final int CONFIG_SAVE_CHECK_TICKS		= CONFIG_SAVE_CHECK_PERIOD/TIMER_PERIOD;
	private static final int CONFIG_SAVE_TICKS				= CONFIG_SAVE_PERIOD/TIMER_PERIOD;
	private static final int PUBLISH_CHECK_PERIOD			= 30*1000;
	private static final int PUBLISH_CHECK_TICKS			= PUBLISH_CHECK_PERIOD/TIMER_PERIOD;
	private static final int PUBLISH_SLEEPING_CHECK_PERIOD	= 5*60*1000;
	private static final int PUBLISH_SLEEPING_CHECK_TICKS	= PUBLISH_SLEEPING_CHECK_PERIOD/TIMER_PERIOD;

	private static final int SECONDARY_LOOKUP_PERIOD		= 15*60*1000;
	private static final int SECONDARY_LOOKUP_TICKS			= SECONDARY_LOOKUP_PERIOD/TIMER_PERIOD;
	private static final int REPUBLISH_PERIOD				= 8*60*60*1000;
	private static final int REPUBLISH_TICKS				= REPUBLISH_PERIOD/TIMER_PERIOD;

	private static final int INITIAL_PUBLISH_DELAY	= 3*60*1000;
	private static final int INITIAL_PUBLISH_TICKS	= INITIAL_PUBLISH_DELAY/TIMER_PERIOD;
	

	private static final int CONFIG_DISCARD_MILLIS	= 60*1000;
	
	protected static final byte		NET_NONE	= 0x00;
	protected static final byte		NET_PUBLIC	= 0x01;
	protected static final byte		NET_I2P		= 0x02;
	protected static final byte		NET_TOR		= 0x04;
	
	private static final String[] NET_PUBLIC_ARRAY 			= { AENetworkClassifier.AT_PUBLIC };
	private static final String[] NET_I2P_ARRAY 			= { AENetworkClassifier.AT_I2P };
	private static final String[] NET_TOR_ARRAY 			= { AENetworkClassifier.AT_TOR };
	private static final String[] NET_PUBLIC_AND_I2P_ARRAY 	= { AENetworkClassifier.AT_PUBLIC, AENetworkClassifier.AT_I2P };

	public static synchronized void
	preInitialise(
		AzureusCore		_core )
	{
		core		= _core;
	}
	
	public static synchronized RelatedContentManager
	getSingleton()
	
		throws ContentException
	{
		if ( singleton == null ){
			
			singleton = new RelatedContentManager();
		}
		
		return( singleton );
	}
	
	protected final Object	rcm_lock	= new Object();

	private PluginInterface 				plugin_interface;
	private TorrentAttribute 				ta_networks;
	private TorrentAttribute 				ta_category;
	private DHTPluginInterface				public_dht_plugin;
				
	private volatile Map<Byte,DHTPluginInterface>		i2p_dht_plugin_map = new HashMap<Byte, DHTPluginInterface>();

	private TagManager						tag_manager;
	
	private long	global_random_id = -1;
	
	private LinkedList<DownloadInfo>			pub_download_infos1 	= new LinkedList<DownloadInfo>();
	private LinkedList<DownloadInfo>			pub_download_infos2 	= new LinkedList<DownloadInfo>();
		
	private LinkedList<DownloadInfo>			non_pub_download_infos1 	= new LinkedList<DownloadInfo>();
	private LinkedList<DownloadInfo>			non_pub_download_infos2 	= new LinkedList<DownloadInfo>();
	
	private ByteArrayHashMapEx<DownloadInfo>	download_info_map	= new ByteArrayHashMapEx<DownloadInfo>();
	private Set<String>							download_priv_set	= new HashSet<String>();

	
	private final boolean	enabled;
	
	private int		max_search_level;
	private int		max_results;
	
	private AtomicInteger	temporary_space = new AtomicInteger();
	
	private int publishing_count = 0;
	
	private CopyOnWriteList<RelatedContentManagerListener>	listeners = new CopyOnWriteList<RelatedContentManagerListener>();
	
	private AESemaphore initialisation_complete_sem = new AESemaphore( "RCM:init" );
	
	private ContentCache				content_cache_ref;
	private WeakReference<ContentCache>	content_cache;
	
	private boolean		content_dirty;
	private long		last_config_access;		
	private int			content_discard_ticks;
	
	private AtomicInteger	total_unread = new AtomicInteger( COConfigurationManager.getIntParameter( CONFIG_TOTAL_UNREAD, 0 ));
	
	private AsyncDispatcher	content_change_dispatcher 	= new AsyncDispatcher();
	
	private static final int SECONDARY_LOOKUP_CACHE_MAX = 10;
	
	private LinkedList<SecondaryLookup> secondary_lookups = new LinkedList<SecondaryLookup>();
	
	private boolean	secondary_lookup_in_progress;
	private long	secondary_lookup_complete_time;
	
	private final 	RelatedContentSearcher	searcher = new RelatedContentSearcher( this );
	
	
	private static final int MAX_TRANSIENT_CACHE	= 256;
	
	protected static Map<String,DownloadInfo> transient_info_cache =
		new LinkedHashMap<String,DownloadInfo>(MAX_TRANSIENT_CACHE,0.75f,true)
		{
			protected boolean 
			removeEldestEntry(
		   		Map.Entry<String,DownloadInfo> eldest) 
			{
				return size() > MAX_TRANSIENT_CACHE;
			}
		};
				
			
	private boolean	persist;
	
	{
		COConfigurationManager.addAndFireParameterListener(
			"rcm.persist",
			new ParameterListener()
			{
				public void 
				parameterChanged(
					String parameterName )
				{
					persist = COConfigurationManager.getBooleanParameter( "rcm.persist" ) || true;
				}
			});
		
			// remove one day
		
		COConfigurationManager.removeParameter( "rcm.dlinfo.history" );
	}
	
	protected
	RelatedContentManager()
	
		throws ContentException
	{
		COConfigurationManager.addAndFireParameterListeners(
				new String[]{
					"rcm.ui.enabled",
					"rcm.max_search_level",
					"rcm.max_results",
				},
				new ParameterListener()
				{
					public void 
					parameterChanged(
						String name )
					{
						max_search_level 	= COConfigurationManager.getIntParameter( "rcm.max_search_level", 3 );
						max_results		 	= COConfigurationManager.getIntParameter( "rcm.max_results", 500 );
					}
				});
		
		if ( !FeatureAvailability.isRCMEnabled() || 
			 !COConfigurationManager.getBooleanParameter( "rcm.overall.enabled", true )){
			
			enabled		= false;
			
			deleteRelatedContent();
			
			initialisation_complete_sem.releaseForever();

			return;
		}
		
		enabled = true;
		
		try{
			if ( core == null ){
				
				throw( new ContentException( "getSingleton called before pre-initialisation" ));
			}
			
			while( global_random_id == -1 ){
				
				global_random_id = COConfigurationManager.getLongParameter( "rcm.random.id", -1 );
				
				if ( global_random_id == -1 ){
					
					global_random_id = RandomUtils.nextLong();
					
					COConfigurationManager.setParameter( "rcm.random.id", global_random_id );
				}
			}
				
			plugin_interface = core.getPluginManager().getDefaultPluginInterface();
			
			ta_networks 	= plugin_interface.getTorrentManager().getAttribute( TorrentAttribute.TA_NETWORKS );
			ta_category 	= plugin_interface.getTorrentManager().getAttribute( TorrentAttribute.TA_CATEGORY );

			tag_manager	= TagManagerFactory.getTagManager();
			
			plugin_interface.getUtilities().createDelayedTask(new AERunnable() {
				public void runSupport() {
					SimpleTimer.addEvent(
							"rcm.delay.init",
							SystemTime.getOffsetTime( 15*1000 ),
							new TimerEventPerformer()
							{
								public void 
								perform(
									TimerEvent event )
								{
									delayedInit();
								}
							});
				}
			}).queue();
			
		}catch( Throwable e ){
			
			initialisation_complete_sem.releaseForever();
			
			if ( e instanceof ContentException ){
				
				throw((ContentException)e);
			}
			
			throw( new ContentException( "Initialisation failed", e ));
		}
	}
	
	protected PluginInterface
	getPluginInterface()
	{
		return( plugin_interface );
	}
	
	private void delayedInit() {
		
		plugin_interface.addListener(
			new PluginListener()
			{
				public void
				initializationComplete()
				{
					if ( !persist ){
						
						deleteRelatedContent();
					}
					
					try{
						PluginInterface dht_pi = 
							plugin_interface.getPluginManager().getPluginInterfaceByClass(
										DHTPlugin.class );
			
						if ( dht_pi != null ){
				
							DHTPlugin dp = (DHTPlugin)dht_pi.getPlugin();
							
							public_dht_plugin = dp;

							searcher.setDHTPlugin( dp );
							
							DownloadManager dm = plugin_interface.getDownloadManager();
							
							Download[] downloads = dm.getDownloads();
							
							addDownloads( downloads, true );
							
							dm.addListener(
								new DownloadManagerListener()
								{
									public void
									downloadAdded(
										Download	download )
									{
										addDownloads( new Download[]{ download }, false );
									}
									
									public void
									downloadRemoved(
										Download	download )
									{
									}
								},
								false );
							
							SimpleTimer.addPeriodicEvent(
								"RCM:publisher",
								TIMER_PERIOD,
								new TimerEventPerformer()
								{
									private int	tick_count;
									
									public void 
									perform(
										TimerEvent event ) 
									{
										tick_count++;
										
										if ( enabled ){
												
											if ( tick_count >= INITIAL_PUBLISH_TICKS ){
												
												if ( tick_count % ( public_dht_plugin.isSleeping()?PUBLISH_SLEEPING_CHECK_TICKS:PUBLISH_CHECK_TICKS) == 0 ){
												
													publish();
												}
												
												if ( tick_count % SECONDARY_LOOKUP_TICKS == 0 ){

													secondaryLookup();
												}
												
												if ( tick_count % REPUBLISH_TICKS == 0 ){

													republish();
												}
												
												if ( tick_count % CONFIG_SAVE_CHECK_TICKS == 0 ){
													
													saveRelatedContent( tick_count );
												}
											}							
										}
										
										searcher.timerTick( enabled, tick_count );
									}		
								});
						}										
					}finally{
							
						initialisation_complete_sem.releaseForever();
					}
				}
				
				public void
				closedownInitiated()
				{
					saveRelatedContent( 0 );
				}
				
				public void
				closedownComplete()
				{
				}
			});
	}
	
	public boolean
	isEnabled()
	{
		return( enabled );
	}
		
	public int
	getMaxSearchLevel()
	{
		return( max_search_level );
	}
	
	public void
	setMaxSearchLevel(
		int		_level )
	{
		COConfigurationManager.setParameter( "rcm.max_search_level", _level );
	}
	
	public int
	getMaxResults()
	{
		return( max_results );
	}
	
	public void
	setMaxResults(
		int		_max )
	{
		COConfigurationManager.setParameter( "rcm.max_results", _max );
		
		enforceMaxResults( false );
	}
	
	private DHTPluginInterface
	selectDHT(
		byte		networks )
	{
		DHTPluginInterface	result = null;
		
		if ((networks & NET_PUBLIC ) != 0 ){
			
			result = public_dht_plugin;
			
		}else if ((networks & NET_I2P ) != 0 ){
			
			synchronized( i2p_dht_plugin_map ){
			
				result = i2p_dht_plugin_map.get( networks );
			
				if ( result == null && !i2p_dht_plugin_map.containsKey( networks )){
	
					try{
						
						List<DistributedDatabase> ddbs = DDBaseImpl.getDDBs( convertNetworks( networks ));
						
						for ( DistributedDatabase ddb: ddbs ){
							
							if ( ddb.getNetwork() == AENetworkClassifier.AT_I2P ){
								
								result = ((DDBaseImpl)ddb).getDHTPlugin();
							}
						}
					}finally{
						
						i2p_dht_plugin_map.put( networks, result );
					}
				}
			}
		}
		
		if ( result != null ){
			
			if ( !result.isEnabled()){
				
				result = null;
			}
		}
		
		return( result );
	}
	
	protected void
	addDownloads(
		Download[]		downloads,
		boolean			initialising )
	{
		synchronized( rcm_lock ){
	
			List<DownloadInfo>	new_info = new ArrayList<DownloadInfo>( downloads.length );
			
			for ( Download download: downloads ){
				
				try{
					if ( !download.isPersistent()){
						
						continue;
					}
					
					Torrent	torrent = download.getTorrent();
	
					if ( torrent == null ){
						
						continue;
					}
					
					byte[]	hash = torrent.getHash();

					if ( download_info_map.containsKey( hash )){
						
						continue;
					}
					
					byte nets = getNetworks( download );

					if ( nets == NET_NONE ){
						
						continue;
					}
					
					TOTorrent to_torrent = PluginCoreUtils.unwrap( torrent );
					
					if ( !TorrentUtils.isReallyPrivate( to_torrent )){
						
						DownloadManagerState state = PluginCoreUtils.unwrap( download ).getDownloadState();

						if ( state.getFlag(DownloadManagerState.FLAG_LOW_NOISE )){
							
							continue;
						}
						
						LinkedList<DownloadInfo>			download_infos1;
						LinkedList<DownloadInfo>			download_infos2;

						if (( nets & NET_PUBLIC ) != 0 ){
							
							download_infos1 	= pub_download_infos1;
							download_infos2 	= pub_download_infos2;
							
						}else{
							
							download_infos1		= non_pub_download_infos1;
							download_infos2 	= non_pub_download_infos2;
						}
						
						long rand = global_random_id ^ state.getLongParameter( DownloadManagerState.PARAM_RANDOM_SEED );						
						
						long cache = state.getLongAttribute( DownloadManagerState.AT_SCRAPE_CACHE );

						int	seeds_leechers;
						
						if ( cache == -1 ){
							
							seeds_leechers = -1;
							
						}else{
							
							int seeds 		= (int)((cache>>32)&0x00ffffff);
							int leechers 	= (int)(cache&0x00ffffff);
							
							seeds_leechers 	= (int)((seeds<<16)|(leechers&0xffff));
						}

						byte[][] keys = getKeys( download );
						
						DownloadInfo info = 
							new DownloadInfo(
								hash,
								hash,
								download.getName(),
								(int)rand,
								torrent.isPrivate()?StringInterner.intern(torrent.getAnnounceURL().getHost()):null,
								keys[0],
								keys[1],
								getTags( download ),
								nets,
								0,
								false,
								torrent.getSize(),
								(int)( to_torrent.getCreationDate()/(60*60)),
								seeds_leechers,
								(byte)PlatformTorrentUtils.getContentNetworkID( to_torrent ));
						
						new_info.add( info );
						
						if ( initialising || download_infos1.size() == 0 ){
							
							download_infos1.add( info );
							
						}else{
							
							download_infos1.add( RandomUtils.nextInt( download_infos1.size()), info );
						}
						
						download_infos2.add( info );
						
						download_info_map.put( hash, info );
						
						if ( info.getTracker() != null ){
							
							download_priv_set.add( getPrivateInfoKey( info ));
						}
					}
				}catch( Throwable e ){
					
					Debug.out( e );
				}
			}
			
			List<Map<String,Object>> history = (List<Map<String,Object>>)COConfigurationManager.getListParameter( "rcm.dlinfo.history.privx", new ArrayList<Map<String,Object>>());
			
			if ( initialising ){
		
				int padd = MAX_HISTORY - download_info_map.size();
				
				for ( int i=0;i<history.size() && padd > 0;i++ ){
					
					try{
						DownloadInfo info = deserialiseDI((Map<String,Object>)history.get(i), null);
						
						if ( info != null && !download_info_map.containsKey( info.getHash())){
							
							download_info_map.put( info.getHash(), info );
							
							if ( info.getTracker() != null ){
								
								download_priv_set.add( getPrivateInfoKey( info ));
							}
							
							if (( info.getNetworksInternal() & NET_PUBLIC ) != 0 ){
							
								pub_download_infos1.add( info );
								pub_download_infos2.add( info );
							
							}else{
								
								non_pub_download_infos1.add( info );
								non_pub_download_infos2.add( info );
							}
							
							padd--;
						}
					}catch( Throwable e ){
						
					}
				}
				
				Collections.shuffle( pub_download_infos1 );
				Collections.shuffle( non_pub_download_infos1 );
				
			}else{
				
				if ( new_info.size() > 0 ){
					
					final List<String>	base32_hashes = new ArrayList<String>();
					
					for ( DownloadInfo info: new_info ){
						
						byte[] hash = info.getHash();
						
						if ( hash != null ){
							
							base32_hashes.add( Base32.encode( hash ));
						}
						
						Map<String,Object> map = serialiseDI( info, null );
							
						if ( map != null ){
							
							history.add( map );	
						}
					}
					
					while( history.size() > MAX_HISTORY ){
						
						history.remove(0);
					}
					
					COConfigurationManager.setParameter( "rcm.dlinfo.history.privx", history );
					
					if ( base32_hashes.size() > 0 ){
						
						content_change_dispatcher.dispatch(
							new AERunnable()
							{
								public void
								runSupport()
								{
									List<RelatedContent>	to_remove = new ArrayList<RelatedContent>();
									
									synchronized( rcm_lock ){
										
										ContentCache content_cache = loadRelatedContent();
										
										for ( String h: base32_hashes ){
										
											DownloadInfo di = content_cache.related_content.get( h );
											
											if ( di != null ){
												
												to_remove.add( di );
											}
										}
									}
									
									if ( to_remove.size() > 0 ){
										
										delete( to_remove.toArray( new RelatedContent[ to_remove.size()] ));
									}
								}
							});
					}
				}
			}
		}
	}
	
	protected void
	republish()
	{
		synchronized( rcm_lock ){

			if ( publishing_count > 0 ){
				
				return;
			}
			
			if ( pub_download_infos1.isEmpty()){
				
				List<DownloadInfo> list = download_info_map.values();
				
				for ( DownloadInfo info: list ){
					
					if (( info.getNetworksInternal() & NET_PUBLIC ) != 0 ){
			
						pub_download_infos1.add( info );
						pub_download_infos2.add( info );
					}
				}
				
				Collections.shuffle( pub_download_infos1 );
			}
			
			if ( non_pub_download_infos1.isEmpty()){
				
				List<DownloadInfo> list = download_info_map.values();
				
				for ( DownloadInfo info: list ){
					
					if (( info.getNetworksInternal() & NET_PUBLIC ) == 0 ){
				
						non_pub_download_infos1.add( info );
						non_pub_download_infos2.add( info );
					}
				}
				
				Collections.shuffle( non_pub_download_infos1 );
			}
		}
	}
	
	private boolean last_pub_was_pub;
	
	protected void
	publish()
	{
		while( true ){
			
			DownloadInfo	info1 = null;
			DownloadInfo	info2 = null;

			synchronized( rcm_lock ){
	
				if ( publishing_count >= MAX_CONCURRENT_PUBLISH ){
					
						// too busy
					
					return;
				}
				
				if ( download_info_map.size() == 1 ){
					
						// only one download, nothing to pair up with
					
					return;
				}
				
				boolean	pub_ok = false;
				
				if ( 	pub_download_infos1.isEmpty() ||
						( 	pub_download_infos1.size() == 1 && 
							pub_download_infos1.getFirst() == pub_download_infos2.getFirst())){
					
					// either none or only one download remaining to be published and it has no partner
					
				}else{
					
					pub_ok = true;
				}
				
				boolean	non_pub_ok = false;
				
				if ( 	non_pub_download_infos1.isEmpty() ||
						( 	non_pub_download_infos1.size() == 1 && 
							non_pub_download_infos1.getFirst() == non_pub_download_infos2.getFirst())){
					
					// either none or only one download remaining to be published and it has no partner
					
				}else{
					
					non_pub_ok = true;
				}
				
				if ( !( pub_ok || non_pub_ok )){
					
					return;
				}
				
				LinkedList<DownloadInfo> download_infos1;
				LinkedList<DownloadInfo> download_infos2;
				
				if ( pub_ok && non_pub_ok ){
				
					if ( last_pub_was_pub ){
						
						pub_ok = false;
					}
					
					last_pub_was_pub = !last_pub_was_pub;
				}
				
				if ( pub_ok ){
					
					download_infos1 = pub_download_infos1;
					download_infos2 = pub_download_infos2;
					
				}else{
					
					download_infos1 = non_pub_download_infos1;
					download_infos2 = non_pub_download_infos2;
				}
				
				if ( download_infos1.isEmpty() || download_info_map.size() == 1 ){
					
					return;
				}
							
				info1 = download_infos1.removeFirst();
				
				Iterator<DownloadInfo> it = download_infos2.iterator();
				
				while( it.hasNext()){
					
					info2 = it.next();
					
					if ( info1 != info2 || download_infos2.size() == 1 ){
						
						it.remove();
						
						break;
					}
				}
				
				if ( info1 == info2 ){
									
					return;
				}
				
				publishing_count++;
			}
			
			try{
				if ( !publish( info1, info2 )){
				
					synchronized( rcm_lock ){

						publishing_count--;
					}
				}				
			}catch( Throwable e ){
				
				synchronized( rcm_lock ){

					publishing_count--;
				}
				
				Debug.out( e );
			}
		}
	}
	
	protected void
	publishNext()
	{
		synchronized( rcm_lock ){

			publishing_count--;
			
			if ( publishing_count < 0 ){
				
					// shouldn't happen but whatever
				
				publishing_count = 0;
			}
		}
		
		publish();
	}
	
	protected boolean
	publish(
		final DownloadInfo	from_info,
		final DownloadInfo	to_info )
	
		throws Exception
	{		
		final DHTPluginInterface dht_plugin = selectDHT( from_info.getNetworksInternal());

		// System.out.println( "publish: " + from_info.getString() + " -> " + to_info.getString() + ": " + dht_plugin );
		
		if ( dht_plugin == null ){
			
			return( false );
		}
		
		final String from_hash	= ByteFormatter.encodeString( from_info.getHash());
		final String to_hash	= ByteFormatter.encodeString( to_info.getHash());
		
		final byte[] key_bytes	= ( "az:rcm:assoc:" + from_hash ).getBytes( "UTF-8" );
		
		String title = to_info.getTitle(); 
		
		if ( title.length() > MAX_TITLE_LENGTH ){
			
			title = title.substring( 0, MAX_TITLE_LENGTH );
		}
		
		Map<String,Object> map = new HashMap<String,Object>();
		
		map.put( "d", title );
		map.put( "r", new Long( Math.abs( to_info.getRand()%1000 )));
		
		String	tracker = to_info.getTracker();
		
		if ( tracker == null ){
			
			map.put( "h", to_info.getHash());
			
		}else{
			
			map.put( "t", tracker );
		}

		if ( to_info.getLevel() == 0 ){
			
			try{
				Download d = to_info.getRelatedToDownload();
			
				if ( d != null ){
					
					Torrent torrent = d.getTorrent();
					
					if ( torrent != null ){
						
						long cnet = PlatformTorrentUtils.getContentNetworkID( PluginCoreUtils.unwrap( torrent ));
						
						if ( cnet != ContentNetwork.CONTENT_NETWORK_UNKNOWN ){
							
							map.put( "c", new Long( cnet ));
						}
						
						long secs = torrent.getCreationDate();
						
						long hours = secs/(60*60);
						
						if ( hours > 0 ){
							
							map.put( "p", new Long( hours ));
						}
					}
										
					int leechers 	= -1;
					int seeds 		= -1;
					
					long cache = PluginCoreUtils.unwrap( d ).getDownloadState().getLongAttribute( DownloadManagerState.AT_SCRAPE_CACHE );
						
					if ( cache != -1 ){
							
						seeds 		= (int)((cache>>32)&0x00ffffff);
						leechers 	= (int)(cache&0x00ffffff);
					}
					
					if ( leechers > 0 ){
						map.put( "l", new Long( leechers ));
					}
					if ( seeds > 0 ){
						map.put( "z", new Long( seeds ));
					}		
					
					byte[][] keys = getKeys( d );
					
					if ( keys[0] != null ){
						map.put( "k", keys[0] );
					}
					if ( keys[1] != null ){
						map.put( "w", keys[1] );
					}
					
					String[] _tags = getTags( d );
					
					if ( _tags != null ){
						map.put( "g", encodeTags( _tags ));
					}
					
					byte nets = getNetworks( d );
					
					if ( nets != NET_PUBLIC ){
						map.put( "o", new Long( nets&0xff ));
					}
				}
			}catch( Throwable e ){		
			}
		}
				
		final Set<String>	my_tags = new HashSet<String>();
		
		try{
			Download d = from_info.getRelatedToDownload();
		
			if ( d != null ){
				
				String[] _tags = getTags( d );
			
				if ( _tags != null ){
					
					map.put( "b", from_info.getRand() % 100 );
					
					map.put( "m", encodeTags( _tags ));
					
					for ( String tag: _tags ){
						
						my_tags.add( tag );
					}
				}
			}
		}catch( Throwable e ){		
		}
		
		long	size = to_info.getSize();
		
		if ( size != 0 ){
			
			map.put( "s", new Long( size ));
		}
		
		final byte[] map_bytes = BEncoder.encode( map );
		
		//System.out.println( "rcmsize=" + map_bytes.length );
		
		final int max_hits = 30;
				
		dht_plugin.get(
				key_bytes,
				"Content rel test: " + from_hash.substring( 0, 16 ),
				DHTPlugin.FLAG_SINGLE_VALUE,
				max_hits,
				30*1000,
				false,
				false,
				new DHTPluginOperationListener()
				{
					private boolean diversified;
					private int		hits;
					
					private Set<String>	entries = new HashSet<String>();
					
					public void
					starts(
						byte[]				key )
					{
					}
					
					public boolean
					diversified()
					{
						diversified = true;
						
						return( false );
					}
					
					public void
					valueRead(
						DHTPluginContact	originator,
						DHTPluginValue		value )
					{
						try{
							Map<String,Object> map = (Map<String,Object>)BDecoder.decode( value.getValue());							
							
							DownloadInfo info = decodeInfo( map, from_info.getHash(), 1, false, entries );

							try{
								String[] r_tags = decodeTags((byte[]) map.get( "m" ));
								
 								if ( r_tags != null ){
																		
									Long	b = (Long)map.get( "b" );
									
										// don't remove tags from the set that we actually published
									
									if ( b == null || from_info.getRand()%100 != b%100 ){
										
										for ( String tag: r_tags ){
											
											synchronized( my_tags ){
												
												my_tags.remove( tag );
											}
										}
									}
								}
							}catch( Throwable e ){
							}						
							
							if ( info != null ){
							
								analyseResponse( info, null );
							}
						}catch( Throwable e ){							
						}
						
						hits++;
					}
					
					public void
					valueWritten(
						DHTPluginContact	target,
						DHTPluginValue		value )
					{
						
					}
					
					public void
					complete(
						byte[]				key,
						boolean				timeout_occurred )
					{
							// if we have something to say prioritise it somewhat
						
						int f_cutoff = my_tags.size()>0?20:10;

						try{
							boolean	do_it;
							
							// System.out.println( from_hash + ": hits=" + hits + ", div=" + diversified );
							
							if ( diversified || hits >= f_cutoff ){
								
								do_it = false;
								
							}else if ( hits <= f_cutoff / 2 ){
								
								do_it = true;
															
							}else{
														
								do_it = RandomUtils.nextInt( hits - (f_cutoff/2) + 1 ) == 0;
							}
								
							if ( do_it ){
								
								try{
									dht_plugin.put(
											key_bytes,
											"Content rel: " +  from_hash.substring( 0, 16 ) + " -> " + to_hash.substring( 0, 16 ),
											map_bytes,
											DHTPlugin.FLAG_ANON,
											new DHTPluginOperationListener()
											{
												public boolean
												diversified()
												{
													return( true );
												}
												
												public void 
												starts(
													byte[] 				key ) 
												{
												}
												
												public void
												valueRead(
													DHTPluginContact	originator,
													DHTPluginValue		value )
												{
												}
												
												public void
												valueWritten(
													DHTPluginContact	target,
													DHTPluginValue		value )
												{
												}
												
												public void
												complete(
													byte[]				key,
													boolean				timeout_occurred )
												{
													publishNext();
												}
											});
								}catch( Throwable e ){
									
									Debug.printStackTrace(e);
									
									publishNext();
								}
							}else{
								
								publishNext();
							}
						}finally{
							
							checkAlternativePubs( to_info, map_bytes, f_cutoff );
						}
					}
				});
		
		return( true );
	}
		
	private void
	checkAlternativePubs(
		DownloadInfo	to_info,
		final byte[]	map_bytes,
		final int		f_cutoff )
	{
		Download dl = to_info.getRelatedToDownload();
		
		if ( dl != null ){
			
			DiskManagerFileInfo[] files = dl.getDiskManagerFileInfo();
			
			List<Long>	sizes = new ArrayList<Long>();
			
			for ( DiskManagerFileInfo file: files ){
				
				long	size = file.getLength();
				
				if ( size >= FILE_ASSOC_MIN_SIZE ){
					
					sizes.add( size );
				}
			}
			
			final DHTPluginInterface dht_plugin = selectDHT( to_info.getNetworksInternal());
			
			if ( dht_plugin != null && sizes.size() > 0 ){

				try{
					final String to_hash	= ByteFormatter.encodeString( to_info.getHash());
	
					final long selected_size = sizes.get( new Random().nextInt( sizes.size()));
	
					final byte[] key_bytes	= ( "az:rcm:size:assoc:" + selected_size ).getBytes( "UTF-8" );
					
					int	max_hits = 30;
					
					dht_plugin.get(
							key_bytes,
							"Content size rel test: " + to_hash.substring( 0, 16 ),
							DHTPlugin.FLAG_SINGLE_VALUE,
							max_hits,
							30*1000,
							false,
							false,
							new DHTPluginOperationListener()
							{
								private boolean diversified;
								private int		hits;
								
								private Set<String>	entries = new HashSet<String>();
								
								public void
								starts(
									byte[]				key )
								{
								}
								
								public boolean
								diversified()
								{
									diversified = true;
									
									return( false );
								}
								
								public void
								valueRead(
									DHTPluginContact	originator,
									DHTPluginValue		value )
								{		
									hits++;
								}
								
								public void
								valueWritten(
									DHTPluginContact	target,
									DHTPluginValue		value )
								{
									
								}
								
								public void
								complete(
									byte[]				key,
									boolean				timeout_occurred )
								{
									boolean	do_it;
									
									// System.out.println( from_hash + ": hits=" + hits + ", div=" + diversified );
									
									if ( diversified || hits >= f_cutoff ){
										
										do_it = false;
										
									}else if ( hits <= f_cutoff / 2 ){
										
										do_it = true;
																	
									}else{
																
										do_it = RandomUtils.nextInt( hits - ( f_cutoff / 2 ) + 1 ) == 0;
									}
										
									if ( do_it ){
										
										try{
											dht_plugin.put(
												key_bytes,
												"Content size rel: " +  selected_size + " -> " + to_hash.substring( 0, 16 ),
												map_bytes,
												DHTPlugin.FLAG_ANON,
												new DHTPluginOperationListener()
												{
													public boolean
													diversified()
													{
														return( true );
													}
													
													public void 
													starts(
														byte[] 				key ) 
													{
													}
													
													public void
													valueRead(
														DHTPluginContact	originator,
														DHTPluginValue		value )
													{
													}
													
													public void
													valueWritten(
														DHTPluginContact	target,
														DHTPluginValue		value )
													{
													}
													
													public void
													complete(
														byte[]				key,
														boolean				timeout_occurred )
													{
													}
												});
										}catch( Throwable e ){
											
											Debug.printStackTrace(e);
										}
									}
								}
							});
				}catch( Throwable e ){
					
					Debug.out( e);
				}
			}
		}
	}
	
	protected DownloadInfo
	decodeInfo(
		Map				map,
		byte[]			from_hash,	// will be null for those trawled from teh local DHT
		int				level,
		boolean			explicit,
		Set<String>		unique_keys )
	{
		try{
			String	title = new String((byte[])map.get( "d" ), "UTF-8" );
			
			String	tracker	= null;
			
			byte[]	hash 	= (byte[])map.get( "h" );
			
			if ( hash == null ){
				
				tracker = new String((byte[])map.get( "t" ), "UTF-8" );
			}
			
			int	rand = ((Long)map.get( "r" )).intValue();
			
			String	key = title + " % " + rand;
			
			synchronized( unique_keys ){
			
				if ( unique_keys.contains( key )){
					
					return( null );
				}
				
				unique_keys.add( key );
			}
			
			Long	l_size = (Long)map.get( "s" );
			
			long	size = l_size==null?0:l_size.longValue();
			
			Long	cnet	 	= (Long)map.get( "c" );
			Long	published 	= (Long)map.get( "p" );
			Long	leechers 	= (Long)map.get( "l" );
			Long	seeds	 	= (Long)map.get( "z" );
			
			// System.out.println( "p=" + published + ", l=" + leechers + ", s=" + seeds );
			
			int	seeds_leechers;
			
			if ( leechers == null && seeds == null ){
				
				seeds_leechers = -1;
				
			}else if ( leechers == null ){
				
				seeds_leechers = seeds.intValue()<<16;
				
			}else if ( seeds == null ){
				
				seeds_leechers = leechers.intValue()&0xffff;
				
			}else{
				
				seeds_leechers = (seeds.intValue()<<16)|(leechers.intValue()&0xffff);
			}
				
			byte[] tracker_keys = (byte[])map.get( "k" );
			byte[] ws_keys 		= (byte[])map.get( "w" );
			
			if ( tracker_keys != null && tracker_keys.length % 4 != 0 ){
				
				tracker_keys = null;
			}
			
			if ( ws_keys != null && ws_keys.length % 4 != 0 ){
				
				ws_keys = null;
			}
			
			byte[]	_tags = (byte[])map.get( "g" );
			
			String[] tags = decodeTags( _tags );
				
			Long _nets = (Long)map.get( "o" );
			
			byte nets = _nets==null?NET_PUBLIC:_nets.byteValue();
			
			return(
				new DownloadInfo( 
						from_hash, hash, title, rand, tracker, tracker_keys, ws_keys, tags, nets, level, explicit, size, 
						published==null?0:published.intValue(),
						seeds_leechers,
						(byte)(cnet==null?ContentNetwork.CONTENT_NETWORK_UNKNOWN:cnet.byteValue()))); 
			
		}catch( Throwable e ){
			
			return( null );
		}
	}
	
	public void
	lookupAttributes(
		final byte[]							from_hash,
		final RelatedAttributeLookupListener	listener )
	
		throws ContentException
	{
		lookupAttributes( from_hash, new String[]{ AENetworkClassifier.AT_PUBLIC }, listener );
	}
	
	public void
	lookupAttributes(
		final byte[]							from_hash,
		final String[]							networks,
		final RelatedAttributeLookupListener	listener )
	
		throws ContentException
	{
		if ( from_hash == null ){
			
			throw( new ContentException( "hash is null" ));
		}

		if ( 	!initialisation_complete_sem.isReleasedForever() ||
				( public_dht_plugin != null && public_dht_plugin.isInitialising())){
			
			AsyncDispatcher dispatcher = new AsyncDispatcher();
	
			dispatcher.dispatch(
				new AERunnable()
				{
					public void
					runSupport()
					{
						try{
							initialisation_complete_sem.reserve();
							
							lookupAttributesSupport( from_hash, convertNetworks( networks ), listener );
							
						}catch( ContentException e ){
							
							Debug.out( e );
						}
					}
				});
		}else{
			
			lookupAttributesSupport( from_hash, convertNetworks( networks ), listener );
		}
	}
	
	private void
	lookupAttributesSupport(
		final byte[]							from_hash,
		final byte								networks,
		final RelatedAttributeLookupListener	listener )
	
		throws ContentException
	{
		try{
			if ( !enabled ){
				
				throw( new ContentException( "rcm is disabled" ));
			}
			
			final DHTPluginInterface dht_plugin = selectDHT( networks );

			if ( dht_plugin == null ){
				
				throw( new Exception( "DHT Plugin unavailable for networks " + getString( convertNetworks( networks ))));
			}
			
				// really should implement a getNetwork() in DHTPluginInterface...
			
			final String dht_plugin_network = dht_plugin == public_dht_plugin?AENetworkClassifier.AT_PUBLIC:AENetworkClassifier.AT_I2P;
			
			final String from_hash_str	= ByteFormatter.encodeString( from_hash );
			
			final byte[] key_bytes	= ( "az:rcm:assoc:" + from_hash_str ).getBytes( "UTF-8" );
			
			String op_str = "Content attr read: " + from_hash_str.substring( 0, 16 );
			
			dht_plugin.get(
					key_bytes,
					op_str,
					DHTPlugin.FLAG_SINGLE_VALUE,
					512,
					30*1000,
					false,
					true,
					new DHTPluginOperationListener()
					{
						private Set<String>	tags = new HashSet<String>();
						
						public void
						starts(
							byte[]				key )
						{
							if ( listener != null ){
								
								try{
									listener.lookupStart();
									
								}catch( Throwable e ){
									
									Debug.out( e );
								}
								
								ContentCache	content_cache = loadRelatedContent();

								List<DownloadInfo> infos = content_cache.related_content_map.get( from_hash );
								
								if ( infos != null ){
									
									for ( DownloadInfo info: infos ){
										
										String[] l_tags = info.getTags();
										
										if ( l_tags != null ){
											
											for ( String tag: l_tags ){
												
												synchronized( tags ){
													
													if ( tags.contains( tag )){
														
														continue;
													}
													
													tags.add( tag );
												}
												
												try{
													listener.tagFound( tag, dht_plugin_network );
													
												}catch( Throwable e ){
													
													Debug.out( e );
												}
											}
										}
									}
								}
							}
						}
						
						public boolean
						diversified()
						{
							return( true );
						}
						
						public void
						valueRead(
							DHTPluginContact	originator,
							DHTPluginValue		value )
						{
							try{
								Map<String,Object> map = (Map<String,Object>)BDecoder.decode( value.getValue());
								
								String[] r_tags = decodeTags((byte[]) map.get( "m" ));
								
								if ( r_tags != null ){
									
									for ( String tag: r_tags ){
										
										synchronized( tags ){
											
											if ( tags.contains( tag )){
												
												continue;
											}
											
											tags.add( tag );
										}
										
										try{
											listener.tagFound( tag, dht_plugin_network );
											
										}catch( Throwable e ){
											
											Debug.out( e );
										}
									}
								}
								
							}catch( Throwable e ){	
							}
						}
						
						public void
						valueWritten(
							DHTPluginContact	target,
							DHTPluginValue		value )
						{
							
						}
						
						public void
						complete(
							byte[]				key,
							boolean				timeout_occurred )
						{
							if ( listener != null ){
								
								try{
									listener.lookupComplete();
									
								}catch( Throwable e ){
									
									Debug.out( e );
								}
							}
						}				
					});
		}catch( Throwable e ){
		
			ContentException	ce;
			
			if ( ( e instanceof ContentException )){
				
				ce = (ContentException)e;
				
			}else{
				ce = new ContentException( "Lookup failed", e );
			}
			
			if ( listener != null ){
				
				try{
					listener.lookupFailed( ce );
					
				}catch( Throwable f ){
					
					Debug.out( f );
				}
			}
			
			throw( ce );
		}
	}
	
	public void
	lookupContent(
		final byte[]						hash,
		final RelatedContentLookupListener	listener )
	
		throws ContentException
	{			
		if ( hash == null ){
			
			throw( new ContentException( "hash is null" ));
		}

		byte	net = NET_PUBLIC;
		
		try{
			Download download = plugin_interface.getDownloadManager().getDownload( hash );
		
			if ( download != null ){
				
				net = getNetworks( download );
			}
		}catch( Throwable e ){
			
		}
		
		final byte f_net = net;
		
		if ( 	!initialisation_complete_sem.isReleasedForever() ||
				( public_dht_plugin != null && public_dht_plugin.isInitialising())){
			
			AsyncDispatcher dispatcher = new AsyncDispatcher();
	
			dispatcher.dispatch(
				new AERunnable()
				{
					public void
					runSupport()
					{
						try{
							initialisation_complete_sem.reserve();
							
							lookupContentSupport( hash, 0, f_net, true, listener );
							
						}catch( ContentException e ){
							
							Debug.out( e );
						}
					}
				});
		}else{
			
			lookupContentSupport( hash, 0, f_net, true, listener );
		}
	}
	
	public void
	lookupContent(
		final byte[]						hash,
		final String[]						networks,
		final RelatedContentLookupListener	listener )
	
		throws ContentException
	{			
		if ( hash == null ){
			
			throw( new ContentException( "hash is null" ));
		}

		final byte	net = convertNetworks( networks );
		
		if ( net == 0 ){
			
			throw( new ContentException( "No networks specified" ));
		}
		
		if ( 	!initialisation_complete_sem.isReleasedForever() ||
				( public_dht_plugin != null && public_dht_plugin.isInitialising())){
			
			AsyncDispatcher dispatcher = new AsyncDispatcher();
	
			dispatcher.dispatch(
				new AERunnable()
				{
					public void
					runSupport()
					{
						try{
							initialisation_complete_sem.reserve();
							
							lookupContentSupport( hash, 0, net, true, listener );
							
						}catch( ContentException e ){
							
							Debug.out( e );
						}
					}
				});
		}else{
			
			lookupContentSupport( hash, 0, net, true, listener );
		}
	}
	
	public void
	lookupContent(
		final long							file_size,
		final RelatedContentLookupListener	listener )
	
		throws ContentException
	{
		if ( file_size < FILE_ASSOC_MIN_SIZE ){
			
			throw( new ContentException( "file size is invalid - min=" + FILE_ASSOC_MIN_SIZE ));
		}

		if ( 	!initialisation_complete_sem.isReleasedForever() ||
				( public_dht_plugin != null && public_dht_plugin.isInitialising())){
			
			AsyncDispatcher dispatcher = new AsyncDispatcher();
	
			dispatcher.dispatch(
				new AERunnable()
				{
					public void
					runSupport()
					{
						try{
							initialisation_complete_sem.reserve();
							
							lookupContentSupport( file_size, NET_PUBLIC, listener );
							
						}catch( ContentException e ){
							
							Debug.out( e );
						}
					}
				});
		}else{
			
			lookupContentSupport( file_size, NET_PUBLIC, listener );
		}
	}
	
	public void
	lookupContent(
		final long							file_size,
		final String[]						networks,
		final RelatedContentLookupListener	listener )
	
		throws ContentException
	{
		if ( file_size < FILE_ASSOC_MIN_SIZE ){
			
			throw( new ContentException( "file size is invalid - min=" + FILE_ASSOC_MIN_SIZE ));
		}

		final byte	net = convertNetworks( networks );
		
		if ( net == 0 ){
			
			throw( new ContentException( "No networks specified" ));
		}

		if ( 	!initialisation_complete_sem.isReleasedForever() ||
				( public_dht_plugin != null && public_dht_plugin.isInitialising())){
			
			AsyncDispatcher dispatcher = new AsyncDispatcher();
	
			dispatcher.dispatch(
				new AERunnable()
				{
					public void
					runSupport()
					{
						try{
							initialisation_complete_sem.reserve();
							
							lookupContentSupport( file_size, net, listener );
							
						}catch( ContentException e ){
							
							Debug.out( e );
						}
					}
				});
		}else{
			
			lookupContentSupport( file_size, net, listener );
		}
	}
	
	private void
	lookupContentSupport(
		final long							file_size,
		final byte							networks,
		final RelatedContentLookupListener	listener )
	
		throws ContentException
	{
		if ( !enabled ){
			
			throw( new ContentException( "rcm is disabled" ));
		}
	
		try{
			final byte[] key_bytes	= ( "az:rcm:size:assoc:" + file_size ).getBytes( "UTF-8" );

				// we need something to use
			
			final byte[] from_hash = new SHA1Simple().calculateHash( key_bytes );
			
			String op_str = "Content rel read: size=" + file_size;

			lookupContentSupport0( from_hash, key_bytes, op_str, 0, networks, true, listener );
			
		}catch( ContentException e ){
			
			throw( e );
			
		}catch( Throwable e ){
			
			throw( new ContentException( "lookup failed", e ));
		}
	}
	
	private void
	lookupContentSupport(
		final byte[]						from_hash,
		final int							level,
		final byte							networks,
		final boolean						explicit,
		final RelatedContentLookupListener	listener )
	
		throws ContentException
	{
		if ( !enabled ){
		
			throw( new ContentException( "rcm is disabled" ));
		}
		
		try{
			
			final String from_hash_str	= ByteFormatter.encodeString( from_hash );
		
			final byte[] key_bytes	= ( "az:rcm:assoc:" + from_hash_str ).getBytes( "UTF-8" );
			
			String op_str = "Content rel read: " + from_hash_str.substring( 0, 16 );

			lookupContentSupport0( from_hash, key_bytes, op_str, level, networks, explicit, listener );
			
		}catch( ContentException e ){
			
			throw( e );
			
		}catch( Throwable e ){
			
			throw( new ContentException( "lookup failed", e ));
		}
	}
		
	private String
	getString(
		String[]	args )
	{
		String str = "";
		
		for ( String s: args ){
			
			str += (str.length()==0?"":",") + s;
		}
		
		return( str );
	}
	
	private void
	lookupContentSupport0(
		final byte[]						from_hash,
		final byte[]						key_bytes,
		final String						op_str,
		final int							level,
		final byte							networks,
		final boolean						explicit,
		final RelatedContentLookupListener	listener )
	
		throws ContentException
	{
		try{
			final int max_hits = 30;
			
			DHTPluginInterface dht_plugin = selectDHT( networks );

			if ( dht_plugin == null ){
				
				throw( new Exception( "DHT Plugin unavailable for networks '" + getString( convertNetworks( networks )) + "'" ));
			}
			
			dht_plugin.get(
					key_bytes,
					op_str,
					DHTPlugin.FLAG_SINGLE_VALUE,
					max_hits,
					60*1000,
					false,
					true,
					new DHTPluginOperationListener()
					{
						private Set<String>	entries = new HashSet<String>();
						
						private RelatedContentManagerListener manager_listener = 
							new RelatedContentManagerListener()
							{
							private Set<RelatedContent>	content_list = new HashSet<RelatedContent>();
							
								public void
								contentFound(
									RelatedContent[]	content )
								{
									handle( content );
								}
	
								public void
								contentChanged(
									RelatedContent[]	content )
								{
									handle( content );
								}
								
								public void 
								contentRemoved(
									RelatedContent[] 	content ) 
								{
								}
								
								public void
								contentChanged()
								{									
								}
								
								public void
								contentReset()
								{
								}
								
								private void
								handle(
									RelatedContent[]	content )
								{
									List<RelatedContent>	new_content = new ArrayList<RelatedContent>( content.length );
									
									synchronized( content_list ){
										
										for ( RelatedContent rc: content ){
										
											if ( !content_list.contains( rc )){
											
												new_content.add( rc );
											}
										}
										
										if ( new_content.size() == 0 ){
											
											return;
										}
										
										content_list.addAll( new_content );
									}
									
									listener.contentFound( new_content.toArray( new RelatedContent[new_content.size()] ));
								}
							};
						
						public void
						starts(
							byte[]				key )
						{
							if ( listener != null ){
								
								try{
									listener.lookupStart();
									
								}catch( Throwable e ){
									
									Debug.out( e );
								}
							}
						}
						
						public boolean
						diversified()
						{
							return( true );
						}
						
						public void
						valueRead(
							DHTPluginContact	originator,
							DHTPluginValue		value )
						{
							try{
								Map<String,Object> map = (Map<String,Object>)BDecoder.decode( value.getValue());
								
								DownloadInfo info = decodeInfo( map, from_hash, level+1, explicit, entries );
								
								if ( info != null ){
									
									analyseResponse( info, listener==null?null:manager_listener );
								}
							}catch( Throwable e ){	
							}
						}
						
						public void
						valueWritten(
							DHTPluginContact	target,
							DHTPluginValue		value )
						{
							
						}
						
						public void
						complete(
							byte[]				key,
							boolean				timeout_occurred )
						{
							if ( listener != null ){
								
								try{
									listener.lookupComplete();
									
								}catch( Throwable e ){
									
									Debug.out( e );
								}
							}
						}				
					});
		}catch( Throwable e ){
		
			ContentException	ce;
			
			if ( ( e instanceof ContentException )){
				
				ce = (ContentException)e;
				
			}else{
				ce = new ContentException( "Lookup failed", e );
			}
			
			if ( listener != null ){
				
				try{
					listener.lookupFailed( ce );
					
				}catch( Throwable f ){
					
					Debug.out( f );
				}
			}
			
			throw( ce );
		}
	}
	
	protected void
	popuplateSecondaryLookups(
		ContentCache	content_cache )
	{
		Random rand = new Random();

		secondary_lookups.clear();
		
			// stuff in a couple primarys
		
		List<DownloadInfo> primaries = download_info_map.values();
		
		int	primary_count = primaries.size();
		
		int	primaries_to_add;
		
		if ( primary_count < 2 ){
			
			primaries_to_add = 0;
			
		}else if ( primary_count < 5 ){
			
			if ( rand.nextInt(4) == 0 ){
				
				primaries_to_add = 1;
				
			}else{
				
				primaries_to_add = 0;
			}
		}else if ( primary_count < 10 ){
			
			primaries_to_add = 1;
			
		}else{
			
			primaries_to_add = 2;
		}
		
		if ( primaries_to_add > 0 ){
			
			Set<DownloadInfo> added = new HashSet<DownloadInfo>();
			
			for (int i=0;i<primaries_to_add;i++){
				
				DownloadInfo info = primaries.get( rand.nextInt( primaries.size()));
				
				if ( !added.contains( info )){
					
					added.add( info );
					
					secondary_lookups.addLast(new SecondaryLookup(info.getHash(), info.getLevel(), info.getNetworksInternal()));
				}
			}
		}
		
		Map<String,DownloadInfo>		related_content			= content_cache.related_content;

		Iterator<DownloadInfo> it = related_content.values().iterator();
		
		List<DownloadInfo> secondary_cache_temp = new ArrayList<DownloadInfo>( related_content.size());

		while( it.hasNext()){
			
			DownloadInfo di = it.next();
			
			if ( di.getHash() != null && di.getLevel() < max_search_level ){
					
				secondary_cache_temp.add( di );
			}
		}
						
		final int cache_size = Math.min( secondary_cache_temp.size(), SECONDARY_LOOKUP_CACHE_MAX - secondary_lookups.size());
		
		if ( cache_size > 0 ){
						
			for( int i=0;i<cache_size;i++){
				
				int index = rand.nextInt( secondary_cache_temp.size());
				
				DownloadInfo x = secondary_cache_temp.get( index );
				
				secondary_cache_temp.set( index, secondary_cache_temp.get(i));
				
				secondary_cache_temp.set( i, x );
			}
			
			for ( int i=0;i<cache_size;i++){
				
				DownloadInfo x = secondary_cache_temp.get(i);
				
				secondary_lookups.addLast(new SecondaryLookup(x.getHash(), x.getLevel(), x.getNetworksInternal()));
			}
		}
	}
	
	protected void
	secondaryLookup()
	{
		SecondaryLookup sl;
		
		long	now = SystemTime.getMonotonousTime();
		
		synchronized( rcm_lock ){
			
			if ( secondary_lookup_in_progress ){
				
				return;
			}
		
			if ( now - secondary_lookup_complete_time < SECONDARY_LOOKUP_PERIOD ){
				
				return;
			}
			
			if ( secondary_lookups.size() == 0 ){
			
				ContentCache cc = content_cache==null?null:content_cache.get();

				if ( cc == null ){
					
						// this will populate the cache
					
					cc = loadRelatedContent();
					
				}else{
					
					popuplateSecondaryLookups( cc );
				}
			}

			if ( secondary_lookups.size() == 0 ){
				
				return;
			}
						
			sl = secondary_lookups.removeFirst();

			secondary_lookup_in_progress = true;
		}
		
		try{
			lookupContentSupport( 
				sl.getHash(),
				sl.getLevel(),
				sl.getNetworks(),
				false,
				new RelatedContentLookupListener()
				{
					public void
					lookupStart()
					{	
					}
					
					public void
					contentFound(
						RelatedContent[]	content )
					{	
					}
					
					public void
					lookupComplete()
					{
						next();
					}
					
					public void
					lookupFailed(
						ContentException 	error )
					{
						next();
					}
					
					protected void
					next()
					{
						final SecondaryLookup next_sl;
						
						synchronized( rcm_lock ){
							
							if ( secondary_lookups.size() == 0 ){
								
								secondary_lookup_in_progress = false;
								
								secondary_lookup_complete_time = SystemTime.getMonotonousTime();
								
								return;
								
							}else{
								
								next_sl = secondary_lookups.removeFirst();
							}
						}
						
						final RelatedContentLookupListener listener = this;
						
						SimpleTimer.addEvent(
							"RCM:SLDelay",
							SystemTime.getOffsetTime( 30*1000 ),
							new TimerEventPerformer()
							{
								public void 
								perform(
									TimerEvent event ) 
								{
									try{					
										lookupContentSupport( next_sl.getHash(), next_sl.getLevel(), next_sl.getNetworks(), false, listener );
										
									}catch( Throwable e ){
										
										//Debug.out( e );
										
										synchronized( rcm_lock ){
											
											secondary_lookup_in_progress = false;
											
											secondary_lookup_complete_time = SystemTime.getMonotonousTime();
										}
									}
								}
							});
					}
				});
			
		}catch( Throwable e ){
			
			//Debug.out( e );
			
			synchronized( rcm_lock ){
				
				secondary_lookup_in_progress = false;
				
				secondary_lookup_complete_time = now;
			}
		}
	}
	
	protected void
	contentChanged(
		final DownloadInfo		info )
	{
		setConfigDirty();
		
		content_change_dispatcher.dispatch(
			new AERunnable()
			{
				public void
				runSupport()
				{
					for ( RelatedContentManagerListener l: listeners ){
						
						try{
							l.contentChanged( new RelatedContent[]{ info });
							
						}catch( Throwable e ){
							
							Debug.out( e );
						}
					}
				}
			});
	}
	
	protected void
	contentChanged(
		boolean	is_dirty )
	{
		if ( is_dirty ){
		
			setConfigDirty();
		}
		
		content_change_dispatcher.dispatch(
			new AERunnable()
			{
				public void
				runSupport()
				{
					for ( RelatedContentManagerListener l: listeners ){
						
						try{
							l.contentChanged();
							
						}catch( Throwable e ){
							
							Debug.out( e );
						}
					}
				}
			});
	}
	
	public void
	delete(
		RelatedContent[]	content )
	{
		synchronized( rcm_lock ){
			
			ContentCache content_cache = loadRelatedContent();
			
			delete( content, content_cache, true );
		}
	}
	
	protected void
	delete(
		final RelatedContent[]	content,
		ContentCache			content_cache,
		boolean					persistent )
	{
		if ( persistent ){
		
			addPersistentlyDeleted( content );
		}
		
		Map<String,DownloadInfo> related_content = content_cache.related_content;

		Iterator<DownloadInfo> it = related_content.values().iterator();
		
		while( it.hasNext()){
		
			DownloadInfo di = it.next();
			
			for ( RelatedContent c: content ){
				
				if ( c == di ){
					
					it.remove();
					
					if ( di.isUnread()){
						
						decrementUnread();
					}
				}
			}
		}
		
		ByteArrayHashMapEx<ArrayList<DownloadInfo>> related_content_map = content_cache.related_content_map;
		
		List<byte[]> delete = new ArrayList<byte[]>();
		
		for ( byte[] key: related_content_map.keys()){
			
			ArrayList<DownloadInfo>	infos = related_content_map.get( key );
			
			for ( RelatedContent c: content ){

				if ( infos.remove( c )){
					
					if ( infos.size() == 0 ){
						
						delete.add( key );
						
						break;
					}
				}
			}
		}
		
		for ( byte[] key: delete ){
			
			related_content_map.remove( key );
		}
		
		setConfigDirty();
		
		content_change_dispatcher.dispatch(
				new AERunnable()
				{
					public void
					runSupport()
					{
						for ( RelatedContentManagerListener l: listeners ){
							
							try{
								l.contentRemoved( content );

							}catch( Throwable e ){
								
								Debug.out( e );
							}
						}
					}
				});
	}
	
	protected String
	getPrivateInfoKey(
		RelatedContent		info )
	{
		return( info.getTitle() + ":" + info.getTracker());
	}
	
	protected void
	analyseResponse(
		DownloadInfo						to_info,
		final RelatedContentManagerListener	listener )
	{
		try{			
			synchronized( rcm_lock ){
				
				byte[] target = to_info.getHash();
				
				String	key;
				
				if ( target != null ){
					
					if ( download_info_map.containsKey( target )){
						
							// target refers to downoad we already have
						
						return;
					}
					
					key = Base32.encode( target );
					
				}else{
					
					key = getPrivateInfoKey( to_info );
					
					if ( download_priv_set.contains( key )){
						
							// target refers to downoad we already have
						
						return;
					}
				}
				
				if ( isPersistentlyDeleted( to_info )){
					
					return;
				}
				
				ContentCache	content_cache = loadRelatedContent();
				
				DownloadInfo	target_info = null;
				
				boolean	changed_content = false;
				boolean	new_content 	= false;
				
				
				target_info = content_cache.related_content.get( key );
				
				if ( target_info == null ){
								
					if ( enoughSpaceFor( content_cache, to_info )){
					
						target_info = to_info;

						content_cache.related_content.put( key, target_info );
						
						byte[] from_hash = to_info.getRelatedToHash();
						
						ArrayList<DownloadInfo> links = content_cache.related_content_map.get( from_hash );
						
						if ( links == null ){
							
							links = new ArrayList<DownloadInfo>(1);
							
							content_cache.related_content_map.put( from_hash, links );
						}
						
						links.add( target_info );
						
						links.trimToSize();
						
						target_info.setPublic( content_cache );
						
						if ( secondary_lookups.size() < SECONDARY_LOOKUP_CACHE_MAX ){
							
							byte[]	hash 	= target_info.getHash();
							int		level	= target_info.getLevel();
							
							if ( hash != null && level < max_search_level ){
								
								secondary_lookups.add( new SecondaryLookup( hash, level, target_info.getNetworksInternal()));
							}
						}
						
						new_content = true;
						
					}else{
						
						transient_info_cache.put( key, to_info );
					}
				}else{
					
						// we already know about this, see if new info
					
					changed_content = target_info.addInfo( to_info );
				}

				if ( target_info != null ){
					
					final RelatedContent[]	f_target 	= new RelatedContent[]{ target_info };
					final boolean			f_change	= changed_content;
					
					final boolean something_changed = changed_content || new_content;
							
					if ( something_changed ){
					
						setConfigDirty();
					}
					
					content_change_dispatcher.dispatch(
						new AERunnable()
						{
							public void
							runSupport()
							{
								if ( something_changed ){
									
									for ( RelatedContentManagerListener l: listeners ){
										
										try{
											if ( f_change ){
												
												l.contentChanged( f_target );
												
											}else{
												
												l.contentFound( f_target );
											}
										}catch( Throwable e ){
											
											Debug.out( e );
										}
									}
								}
								
								if ( listener != null ){
									
									try{
										if ( f_change ){
											
											listener.contentChanged( f_target );
											
										}else{
											
											listener.contentFound( f_target );
										}
									}catch( Throwable e ){
										
										Debug.out( e );
									}
								}
							}
						});
				}
			}
			
		}catch( Throwable e ){
			
			Debug.out( e );
		}
	}
	
	protected boolean
	enoughSpaceFor(
		ContentCache	content_cache,
		DownloadInfo	fi )
	{
		Map<String,DownloadInfo> related_content = content_cache.related_content;
		
		if ( related_content.size() < max_results + temporary_space.get()){
			
			return( true );
		}
		
		Iterator<Map.Entry<String,DownloadInfo>>	it = related_content.entrySet().iterator();
				
		int	max_level 		= fi.getLevel();
		
			// delete oldest at highest level >= level with minimum rank
	
		Map<Integer,DownloadInfo>	oldest_per_rank = new HashMap<Integer, DownloadInfo>();
		
		int	min_rank 	= Integer.MAX_VALUE;
		int	max_rank	= -1;
		
		while( it.hasNext()){
			
			Map.Entry<String,DownloadInfo> entry = it.next();
			
			DownloadInfo info = entry.getValue();
			
			if ( info.isExplicit()){
				
				continue;
			}
			
			int	info_level = info.getLevel();
			
			if ( info_level >= max_level ){
				
				if ( info_level > max_level ){
					
					max_level = info_level;
					
					min_rank 	= Integer.MAX_VALUE;
					max_rank	= -1;
					
					oldest_per_rank.clear();
				}
				
				int	rank = info.getRank();
				
				if ( rank < min_rank ){
					
					min_rank = rank;
					
				}else if ( rank > max_rank ){
					
					max_rank = rank;
				}
				
				DownloadInfo oldest = oldest_per_rank.get( rank );
				
				if ( oldest == null ){
					
					oldest_per_rank.put( rank, info );
					
				}else{
					
					if ( info.getLastSeenSecs() < oldest.getLastSeenSecs()){
						
						oldest_per_rank.put( rank, info );
					}
				}
			}
		}
		
		DownloadInfo to_remove = oldest_per_rank.get( min_rank );
		
		if ( to_remove != null ){
					
			delete( new RelatedContent[]{ to_remove }, content_cache, false );
			
			return( true );
		}
		
			// we don't want high-ranked entries to get stuck there and prevent newer stuff from getting in and rising up
		
		if ( max_level == 1 ){
			
			to_remove = oldest_per_rank.get( max_rank );
			
			if ( to_remove != null ){
				
				int	now_secs = (int)( SystemTime.getCurrentTime()/1000 );
				
					// give it a day at the top
				
				if ( now_secs - to_remove.getLastSeenSecs() >= 24*60*60 ){
					
					delete( new RelatedContent[]{ to_remove }, content_cache, false );
					
					return( true );
				}
			}
		}
		
		return( false );
	}
	
	public RelatedContent[]
	getRelatedContent()
	{
		synchronized( rcm_lock ){

			ContentCache	content_cache = loadRelatedContent();
			
			return( content_cache.related_content.values().toArray( new DownloadInfo[ content_cache.related_content.size()]));
		}
	}
	
	protected List<DownloadInfo>
  	getRelatedContentAsList()
  	{
  		synchronized( rcm_lock ){

  			ContentCache	content_cache = loadRelatedContent();
  			
  			return( new ArrayList<DownloadInfo>( content_cache.related_content.values()));
  		}
  	}
	
	public void
	reset()
	{
		reset( true );
	}
	
	protected void
	reset(
		boolean	reset_perm_dels )
	{
		synchronized( rcm_lock ){
			
			ContentCache cc = content_cache==null?null:content_cache.get();
			
			if ( cc == null ){
				
				FileUtil.deleteResilientConfigFile( CONFIG_FILE );
				
			}else{
			
				cc.related_content 		= new HashMap<String,DownloadInfo>();
				cc.related_content_map 	= new ByteArrayHashMapEx<ArrayList<DownloadInfo>>();
			}
					
			pub_download_infos1.clear();
			pub_download_infos2.clear();
			
			non_pub_download_infos1.clear();
			non_pub_download_infos2.clear();
			
			List<DownloadInfo>	list = download_info_map.values();
			
			for ( DownloadInfo info: list ){
				
				if (( info.getNetworksInternal() & NET_PUBLIC ) != 0 ){
			
					pub_download_infos1.add( info );
					pub_download_infos2.add( info );
					
				}else{
					
					non_pub_download_infos1.add( info );
					non_pub_download_infos2.add( info );
				}
			}
			
			Collections.shuffle( pub_download_infos1 );
			Collections.shuffle( non_pub_download_infos1 );
			
			total_unread.set( 0 );
			
			if ( reset_perm_dels ){
			
				resetPersistentlyDeleted();
			}
			
			setConfigDirty();
		}
		
		content_change_dispatcher.dispatch(
				new AERunnable()
				{
					public void
					runSupport()
					{
						for ( RelatedContentManagerListener l: listeners ){
							
							l.contentReset();
						}
					}
				});
	}
		
	public SearchInstance
	searchRCM(
		Map<String,Object>		search_parameters,
		SearchObserver			_observer )
	
		throws SearchException
	{
		initialisation_complete_sem.reserve();

		if ( !enabled ){
			
			throw( new SearchException( "rcm is disabled" ));
		}
		
		return( searcher.searchRCM(search_parameters, _observer));
	}
	

	protected void
	setConfigDirty()
	{
		synchronized( rcm_lock ){
			
			content_dirty	= true;
		}
	}
	
	protected ContentCache
	loadRelatedContent()
	{
		boolean	fire_event = false;
		
		try{
			synchronized( rcm_lock ){
	
				last_config_access = SystemTime.getMonotonousTime();
	
				ContentCache cc = content_cache==null?null:content_cache.get();
				
				if ( cc == null ){
				
					if ( TRACE ){
						System.out.println( "rcm: load new" );
					}
					
					fire_event = true;
					
					cc = new ContentCache();
		
					content_cache = new WeakReference<ContentCache>( cc );
					
					try{
						int	new_total_unread = 0;
		
						if ( FileUtil.resilientConfigFileExists( CONFIG_FILE )){
											
							Map map = FileUtil.readResilientConfigFile( CONFIG_FILE );
							
							Map<String,DownloadInfo>						related_content			= cc.related_content;
							ByteArrayHashMapEx<ArrayList<DownloadInfo>>		related_content_map		= cc.related_content_map;
		
							Map<String,String>	rcm_map;
							
							byte[]	data = (byte[])map.get( "d" );
							
							if ( data != null ){
										
								try{
									map = BDecoder.decode(new BufferedInputStream( new GZIPInputStream( new ByteArrayInputStream( CryptoManagerFactory.getSingleton().deobfuscate( data )))));
									
								}catch( Throwable e ){
									
										// can get here is config's been deleted
									
									map = new HashMap();
								}
							}
							
							rcm_map = (Map<String,String>)map.get( "rcm" );
						
							Object	rc_map_stuff 	= map.get( "rc" );
							
							if ( rc_map_stuff != null && rcm_map != null ){
								
								Map<Integer,DownloadInfo> id_map = new HashMap<Integer, DownloadInfo>();
									
								if ( rc_map_stuff instanceof Map ){
									
										// migration from when it was a Map with non-ascii key issues
									
									Map<String,Map<String,Object>>	rc_map 	= (Map<String,Map<String,Object>>)rc_map_stuff;

									for ( Map.Entry<String,Map<String,Object>> entry: rc_map.entrySet()){
										
										try{
										
											String	key = entry.getKey();
										
											Map<String,Object>	info_map = entry.getValue();
																		
											DownloadInfo info = deserialiseDI( info_map, cc );
											
											if ( info.isUnread()){
												
												new_total_unread++;
											}
											
											related_content.put( key, info );
											
											int	id = ((Long)info_map.get( "_i" )).intValue();
				
											id_map.put( id, info );
											
										}catch( Throwable e ){
											
											Debug.out( e );
										}
									}
								}else{
									
									List<Map<String,Object>>	rc_map_list 	= (List<Map<String,Object>>)rc_map_stuff;

									for ( Map<String,Object> info_map: rc_map_list ){
										
										try{
										
											String	key = new String((byte[])info_map.get( "_k" ), "UTF-8" );
																												
											DownloadInfo info = deserialiseDI( info_map, cc );
											
											if ( info.isUnread()){
												
												new_total_unread++;
											}
											
											related_content.put( key, info );
											
											int	id = ((Long)info_map.get( "_i" )).intValue();
				
											id_map.put( id, info );
											
										}catch( Throwable e ){
											
											Debug.out( e );
										}
									}
								}
															
								if ( rcm_map.size() != 0 && id_map.size() != 0 ){
									
									for ( String key: rcm_map.keySet()){
										
										try{
											byte[]	hash = Base32.decode( key );
											
											int[]	ids = ImportExportUtils.importIntArray( rcm_map, key );
											
											if ( ids == null || ids.length == 0 ){
												
												// Debug.out( "Inconsistent - no ids" );
												
											}else{
												
												ArrayList<DownloadInfo>	di_list = new ArrayList<DownloadInfo>(ids.length);
												
												for ( int id: ids ){
													
													DownloadInfo di = id_map.get( id );
													
													if ( di == null ){
														
														// Debug.out( "Inconsistent: id " + id + " missing" );
														
													}else{
														
															// we don't currently remember all originators, just one that works 
															
														di.setRelatedToHash( hash );
														
														di_list.add( di );
													}
												}
												
												if ( di_list.size() > 0 ){
													
													related_content_map.put( hash, di_list );
												}
											}
										}catch( Throwable e ){
											
											Debug.out( e );
										}
									}
								}
								
								Iterator<DownloadInfo> it = related_content.values().iterator();
								
								while( it.hasNext()){
									
									DownloadInfo di = it.next();
									
									if ( di.getRelatedToHash() == null ){
								
										// Debug.out( "Inconsistent: info not referenced" );
										
										if ( di.isUnread()){
											
											new_total_unread--;
										}
										
										it.remove();
									}
								}
								
								popuplateSecondaryLookups( cc );
							}
						}
						
						if ( total_unread.get() != new_total_unread ){
														
							// Debug.out( "total_unread - inconsistent (" + total_unread + "/" + new_total_unread + ")" );
							
							total_unread.set( new_total_unread );
							
							COConfigurationManager.setParameter( CONFIG_TOTAL_UNREAD, new_total_unread );
						}
					}catch( Throwable e ){
						
						Debug.out( e );
					}
					
					enforceMaxResults( cc, false );
				}
				
				content_cache_ref = cc;
					
				return( cc );
			}
		}finally{
			
			if ( fire_event ){
				
				contentChanged( false );
			}
		}
	}
	
	protected void
	saveRelatedContent(
		int	tick_count )
	{
		synchronized( rcm_lock ){
				
			COConfigurationManager.setParameter( CONFIG_TOTAL_UNREAD, total_unread.get());
			
			long	now = SystemTime.getMonotonousTime();;
			
			ContentCache cc = content_cache==null?null:content_cache.get();
			
			if ( !content_dirty ){
					
				if ( cc != null  ){
					
					if ( now - last_config_access > CONFIG_DISCARD_MILLIS ){
					
						if ( content_cache_ref != null ){
							
							content_discard_ticks = 0;
						}
						
						if ( TRACE ){
							System.out.println( "rcm: discard: tick count=" + content_discard_ticks++ );
						}
						
						content_cache_ref	= null;
					}
				}else{
					
					if ( TRACE ){
						System.out.println( "rcm: discarded" );
					}
				}
				
				return;
			}
			
			if ( tick_count % CONFIG_SAVE_TICKS != 0 ){
				
				return;
			}
			
			last_config_access = now;
			
			content_dirty	= false;
			
			if ( cc == null ){
				
				// Debug.out( "RCM: cache inconsistent" );
				
			}else{
				
				if ( persist ){
					
					if ( TRACE ){
						System.out.println( "rcm: save" );
					}
					
					Map<String,DownloadInfo>						related_content			= cc.related_content;
					ByteArrayHashMapEx<ArrayList<DownloadInfo>>		related_content_map		= cc.related_content_map;
	
					if ( related_content.size() == 0 ){
						
						FileUtil.deleteResilientConfigFile( CONFIG_FILE );
						
					}else{
						
						Map<String,Object>	map = new HashMap<String, Object>();
						
						Set<Map.Entry<String,DownloadInfo>> rcs = related_content.entrySet();
											
						List<Map<String,Object>> rc_map_list = new ArrayList<Map<String, Object>>( rcs.size());
						
						map.put( "rc", rc_map_list );
						
						int		id = 0;
						
						Map<DownloadInfo,Integer>	info_map = new HashMap<DownloadInfo, Integer>();
						
						for ( Map.Entry<String,DownloadInfo> entry: rcs ){
												
							DownloadInfo	info = entry.getValue();
													
							Map<String,Object> di_map = serialiseDI( info, cc );
							
							if ( di_map != null ){
								
								info_map.put( info, id );
	
								di_map.put( "_i", new Long( id ));
								di_map.put( "_k", entry.getKey());
								
								rc_map_list.add( di_map );
		
								id++;	
							}
						}
						
						Map<String,Object> rcm_map = new HashMap<String, Object>();
						
						map.put( "rcm", rcm_map );

						for ( byte[] hash: related_content_map.keys()){
							
							List<DownloadInfo> dis = related_content_map.get( hash );
							
							int[] ids = new int[dis.size()];
							
							int	pos = 0;
							
							for ( DownloadInfo di: dis ){
								
								Integer	index = info_map.get( di );
								
								if ( index == null ){
									
									// Debug.out( "inconsistent: info missing for " + di );
									
									break;
									
								}else{
									
									ids[pos++] = index;
								}
							}
							
							if ( pos == ids.length ){
							
								ImportExportUtils.exportIntArray( rcm_map, Base32.encode( hash), ids );
							}
						}
						
						if ( true ){
						
							ByteArrayOutputStream baos = new ByteArrayOutputStream( 100*1024 );
							
							try{
								GZIPOutputStream gos = new GZIPOutputStream( baos );
								
								gos.write( BEncoder.encode( map ));
								
								gos.close();
								
							}catch( Throwable e ){
								
								Debug.out( e );
							}
							
							map.clear();
							
							map.put( "d", CryptoManagerFactory.getSingleton().obfuscate( baos.toByteArray()));
						}
						
						FileUtil.writeResilientConfigFile( CONFIG_FILE, map );
					}
				}else{
					
					deleteRelatedContent();
				}
			
				searcher.updateKeyBloom( cc );
			}
		}
	}
	
	private void
	deleteRelatedContent()
	{
		FileUtil.deleteResilientConfigFile( CONFIG_FILE );
		FileUtil.deleteResilientConfigFile( PERSIST_DEL_FILE );
	}
	
	public int
	getNumUnread()
	{
		return( total_unread.get());
	}
	
	public void
	setAllRead()
	{
		boolean	changed = false;
		
		synchronized( rcm_lock ){
			
			DownloadInfo[] content = (DownloadInfo[])getRelatedContent();
			
			for ( DownloadInfo c: content ){
				
				if ( c.isUnread()){
				
					changed = true;
					
					c.setUnreadInternal( false );
				}
			}
			
			total_unread.set( 0 );
		}
		
		if ( changed ){
		
			contentChanged( true );
		}
	}
	
	public void
	deleteAll()
	{	
		synchronized( rcm_lock ){

			ContentCache	content_cache = loadRelatedContent();
			
			addPersistentlyDeleted( content_cache.related_content.values().toArray( new DownloadInfo[ content_cache.related_content.size()]));
		
			reset( false );
		}
	}
	
	protected void
	incrementUnread()
	{
		total_unread.incrementAndGet();
	}
	
	protected void
	decrementUnread()
	{
		synchronized( rcm_lock ){
			
			int val = total_unread.decrementAndGet();
			
			if ( val < 0 ){
				
				// Debug.out( "inconsistent" );
				
				total_unread.set( 0 );
			}
		}
	}
	
	protected Download
	getDownload(
		byte[]	hash )
	{
		try{
			return( plugin_interface.getDownloadManager().getDownload( hash ));
			
		}catch( Throwable e ){
			
			return( null );
		}
	}
	
	private byte[][]
	getKeys(
		Download		download )
	{
		byte[]	tracker_keys	= null;
		byte[]	ws_keys			= null;
		
		try{
			Torrent torrent = download.getTorrent();
		
			if ( torrent != null ){
				
				TOTorrent to_torrent = PluginCoreUtils.unwrap( torrent );
				
				Set<String>	tracker_domains = new HashSet<String>();
				
				addURLToDomainKeySet( tracker_domains, to_torrent.getAnnounceURL());
				
				TOTorrentAnnounceURLGroup group = to_torrent.getAnnounceURLGroup();
				
				TOTorrentAnnounceURLSet[] sets = group.getAnnounceURLSets();
				
				for ( TOTorrentAnnounceURLSet set: sets ){
					
					URL[] urls = set.getAnnounceURLs();
					
					for ( URL u: urls ){
						
						addURLToDomainKeySet( tracker_domains, u );
					}
				}
				
				tracker_keys = domainsToArray( tracker_domains, 8 );
				
				Set<String>	ws_domains = new HashSet<String>();

				List getright = BDecoder.decodeStrings( getURLList( to_torrent, "url-list" ));
				List webseeds = BDecoder.decodeStrings( getURLList( to_torrent, "httpseeds" ));
								
				for ( List l: new List[]{ getright, webseeds }){
					
					for ( Object o: l ){
					
						if ( o instanceof String ){
						
							try{
								addURLToDomainKeySet( ws_domains, new URL((String)o));
								
							}catch( Throwable e ){
								
							}
						}
					}
				}
				
				ws_keys = domainsToArray( ws_domains, 3 );
			}
		}catch( Throwable e ){	
		}
		
		return( new byte[][]{ tracker_keys, ws_keys });
	}
	
	protected byte[]
	domainsToArray(
		Set<String>	domains,
		int			max )
	{
		int	entries = Math.min( domains.size(), max );
		
		if ( entries > 0 ){
			
			byte[] keys = new byte[ entries*4 ];
			
			int	pos = 0;
			
			for ( String dom: domains ){
				
				int hash = dom.hashCode();
				
				byte[]	bytes = { (byte)(hash>>24), (byte)(hash>>16),(byte)(hash>>8),(byte)hash };
				
				System.arraycopy( bytes, 0, keys, pos, 4 );
			
				pos += 4;
			}
			
			return( keys );
		}
		
		return( null );
	}
	
	protected List
	getURLList(
		TOTorrent	torrent,
		String		key )
	{
		Object obj = torrent.getAdditionalProperty( key );
		
		if ( obj instanceof byte[] ){
			
            List l = new ArrayList();
            
	        l.add(obj);
	        
	        return( l );
	        
		}else if ( obj instanceof List ){
			
			return (List)BEncoder.clone(obj);
			
		}else{
			
			return( new ArrayList());
		}
	}
	
	private void
	addURLToDomainKeySet(
		Set<String>	set,
		URL			u )
	{
		String prot = u.getProtocol();
		
		if ( prot != null ){
			
			if ( prot.equalsIgnoreCase( "http" ) || prot.equalsIgnoreCase( "udp" )){
				
				String host = u.getHost().toLowerCase( Locale.US );
				
				if ( host.contains( ":" )){
				
						// ipv6 raw
					
					return;
				}
				
				String[] bits = host.split( "\\." );
				
				int	len = bits.length;
				
				if ( len >= 2 ){
					
					String	end = bits[len-1];
					
					char[] chars = end.toCharArray();
					
						// simple check for ipv4 raw
					
					boolean	all_digits = true;
					
					for ( char c: chars ){
						
						if ( !Character.isDigit( c )){
							
							all_digits = false;
							
							break;
						}
					}
					
					if ( !all_digits ){
						
						set.add( bits[len-2] + "." + end );
					}
				}
			}
		}
	}
		
	private byte
	getNetworks(
		Download		download )
	{
		String[]	networks = download.getListAttribute( ta_networks );
		
		if ( networks == null ){
				
			return( NET_NONE );
			
		}else{
		
			return( convertNetworks( networks ));
		}
	}
	
	public static String[]
	convertNetworks(
		byte		net )
	{
		if ( net == NET_NONE ){
			return( new String[0] );
		}else if ( net == NET_PUBLIC ){
			return( NET_PUBLIC_ARRAY );
		}else if ( net == NET_I2P ){
			return( NET_I2P_ARRAY );		
		}else if ( net == NET_TOR ){
			return( NET_TOR_ARRAY );
		}else if ( net == (NET_PUBLIC | NET_I2P )){
			return( NET_PUBLIC_AND_I2P_ARRAY );
		}else{
			List<String>	nets = new ArrayList<String>();
			
			if (( net & NET_PUBLIC ) != 0 ){
				nets.add( AENetworkClassifier.AT_PUBLIC );
			}
			if (( net & NET_I2P ) != 0 ){
				nets.add( AENetworkClassifier.AT_I2P );
			}
			if (( net & NET_TOR ) != 0 ){
				nets.add( AENetworkClassifier.AT_TOR );
			}
			
			return( nets.toArray( new String[ nets.size()]));
		}
	}
	
	public static byte
	convertNetworks(
		String[]		networks )
	{
		byte	nets = NET_NONE;

		for ( int i=0;i<networks.length;i++ ){
			
			String n = networks[i];
			
			if (n.equalsIgnoreCase( AENetworkClassifier.AT_PUBLIC )){
				
				nets |= NET_PUBLIC;
				
			}else if ( n.equalsIgnoreCase( AENetworkClassifier.AT_I2P )){
				
				nets |= NET_I2P;
				
			}else if ( n.equalsIgnoreCase( AENetworkClassifier.AT_TOR )){
				
				nets |= NET_TOR;
			}
		}
		
		return( nets );
	}
	
	private String[]
	getTags(
		Download	download )
	{		
		Set<String>	all_tags = new HashSet<String>();
	
		if ( tag_manager.isEnabled()){

			String	cat_name = ta_category==null?null:download.getAttribute( ta_category );
			
			if ( cat_name != null ){
				
				Tag cat_tag = tag_manager.getTagType( TagType.TT_DOWNLOAD_CATEGORY ).getTag( cat_name, true );
				
				if ( cat_tag != null && cat_tag.isPublic()){
				
					all_tags.add( cat_name.toLowerCase( Locale.US ));
				}
			}
			
			List<Tag> tags = tag_manager.getTagType( TagType.TT_DOWNLOAD_MANUAL ).getTagsForTaggable( PluginCoreUtils.unwrap( download ));
			
			for ( Tag t: tags ){
				
				if ( t.isPublic()){
				
					all_tags.add( t.getTagName( true ).toLowerCase( Locale.US ));
				}
			}
		}
		
		String[]	networks = download.getListAttribute( ta_networks );

		for ( String network: networks ){
			
			if ( !network.equals( "Public" )){
				
				if ( AEPluginProxyHandler.hasPluginProxyForNetwork( network, true )){
				
					all_tags.add( "_" + network.toLowerCase( Locale.US ) + "_" );
				}
			}
		}
		
		if ( all_tags.size() == 0 ){
		
			return( null );
			
		}else if ( all_tags.size() == 1 ){
			
			return( new String[]{ all_tags.iterator().next()});
			
		}else{
			
			List<String> temp = new ArrayList<String>( all_tags );
			
			Collections.shuffle( temp );
			
			return( temp.toArray( new String[ temp.size()] ));
		}
	}
	
	private static final int MAX_TAGS 			= 3;
	private static final int MAX_TAG_LENGTH		= 20;
	
	protected byte[]
	encodeTags(
		String[]		tags )
	{
		if ( tags == null || tags.length == 0 ){
			
			return( null );
		}
		
		byte[]	temp 	= new byte[MAX_TAGS*(MAX_TAG_LENGTH+1)];
		int		pos		= 0;
		
		for ( int i=0;i<Math.min( MAX_TAGS, tags.length);i++){
			
			String tag = tags[i];
			
			tag = truncateTag( tag );
			
			try{
				byte[] tag_bytes = tag.getBytes( "UTF-8" );
					
				int	tb_len = tag_bytes.length;
											
				temp[pos++] = (byte)tb_len;
						
				System.arraycopy( tag_bytes, 0, temp, pos, tb_len );
						
				pos += tb_len;
				
			}catch( Throwable e ){

			}
		}
		
		if ( pos == 0 ){
			
			return( null );
			
		}else{
			
			byte[] result = new byte[pos];
			
			System.arraycopy( temp, 0, result, 0, pos );
			
			return( result );
		}
	}
	
	protected String
	truncateTag(
		String	tag )
	{
		if ( tag.length() > MAX_TAG_LENGTH ){
			
			tag = tag.substring( 0, MAX_TAG_LENGTH );
		}
		
		while( tag.length() > 0 ){
			
			try{
				byte[] tag_bytes = tag.getBytes( "UTF-8" );
								
				if ( tag_bytes.length <= MAX_TAG_LENGTH ){
					
					break;
					
				}else{
					
					tag = tag.substring( 0, tag.length() - 1 );
				}
				
			}catch( Throwable e ){
				
				break;
			}
		}
		
		return( tag );
	}
	
	protected String[]
	decodeTags(
		byte[]		bytes )
	{
		if ( bytes == null || bytes.length == 0 ){
		
			return( null );
		}
		
		List<String>	tags = new ArrayList<String>( MAX_TAGS );
		
		int	pos = 0;
		
		while( pos < bytes.length ){
			
			int	tag_len = bytes[pos++]&0x000000ff;
			
			if ( tag_len > MAX_TAG_LENGTH ){
				
				break;
			}
			
			try{
				tags.add( new String( bytes, pos, tag_len, "UTF-8" ));
				
				pos += tag_len;
				
			}catch( Throwable e ){
				
				break;
			}
		}
		
		if ( tags.size() == 0 ){
		
			return( null );
			
		}else{
				
			return( tags.toArray( new String[ tags.size()] ));
		}
	}
	
	private static final int PD_BLOOM_INITIAL_SIZE		= 1000;
	private static final int PD_BLOOM_INCREMENT_SIZE	= 1000;
	
	
	private BloomFilter	persist_del_bloom;
	
	protected byte[]
	getPermDelKey(
		RelatedContent	info )
	{
		byte[]	bytes = info.getHash();
		
		if ( bytes == null ){
			
			try{
				bytes = new SHA1Simple().calculateHash( getPrivateInfoKey(info).getBytes( "ISO-8859-1" ));
				
			}catch( Throwable e ){
				
				Debug.out( e );
				
				return( null );
			}
		}
		
		byte[] key = new byte[8];
		
		System.arraycopy( bytes, 0, key, 0, 8 );
		
		return( key );
	}
	
	protected List<byte[]>
	loadPersistentlyDeleted()
	{
		List<byte[]> entries = null;
		
		if ( FileUtil.resilientConfigFileExists( PERSIST_DEL_FILE )){
				
			Map<String,Object> map = (Map<String,Object>)FileUtil.readResilientConfigFile( PERSIST_DEL_FILE );
				
			entries = (List<byte[]>)map.get( "entries" );
		}
	
		if ( entries == null ){
			
			entries = new ArrayList<byte[]>(0);
		}
		
		return( entries );
	}
	
	protected void
	addPersistentlyDeleted(
		RelatedContent[]	content )
	{		
		if ( content.length == 0 ){
			
			return;
		}
	
		List<byte[]> entries = loadPersistentlyDeleted();
		
		List<byte[]> new_keys = new ArrayList<byte[]>( content.length );
		
		for ( RelatedContent rc: content ){
			
			byte[] key = getPermDelKey( rc );
			
			new_keys.add( key );
			
			entries.add( key );
		}
		
		Map<String,Object>	map = new HashMap<String, Object>();
		
		map.put( "entries", entries );
		
		FileUtil.writeResilientConfigFile( PERSIST_DEL_FILE, map );
		
		if ( persist_del_bloom != null ){
			
			if ( persist_del_bloom.getSize() / ( persist_del_bloom.getEntryCount() + content.length ) < 10 ){
		
				persist_del_bloom = BloomFilterFactory.createAddOnly( Math.max( PD_BLOOM_INITIAL_SIZE, persist_del_bloom.getSize() *10 + PD_BLOOM_INCREMENT_SIZE + content.length  ));
				
				for ( byte[] k: entries ){
					
					persist_del_bloom.add( k );
				}
			}else{
				
				for ( byte[] k: new_keys ){
					
					persist_del_bloom.add( k );
				}
			}
		}
	}
	
	protected boolean
	isPersistentlyDeleted(
		RelatedContent		content )
	{
		if ( persist_del_bloom == null ){
			
			List<byte[]> entries = loadPersistentlyDeleted();
			
			persist_del_bloom = BloomFilterFactory.createAddOnly( Math.max( PD_BLOOM_INITIAL_SIZE, entries.size()*10 + PD_BLOOM_INCREMENT_SIZE ));

			for ( byte[] k: entries ){
				
				persist_del_bloom.add( k );
			}
		}
		
		byte[]	key = getPermDelKey( content );
		
		return( persist_del_bloom.contains( key ));
	}

	protected void
	resetPersistentlyDeleted()
	{
		FileUtil.deleteResilientConfigFile( PERSIST_DEL_FILE );
		
		persist_del_bloom = BloomFilterFactory.createAddOnly( PD_BLOOM_INITIAL_SIZE );
	}
	
	public void
	reserveTemporarySpace()
	{
		temporary_space.addAndGet( TEMPORARY_SPACE_DELTA );
	}
	
	public void
	releaseTemporarySpace()
	{
		boolean	reset_explicit = temporary_space.addAndGet( -TEMPORARY_SPACE_DELTA ) == 0;
		
		enforceMaxResults( reset_explicit );
	}
	
	protected void
	enforceMaxResults(
		boolean reset_explicit )
	{
		synchronized( rcm_lock ){
	
			ContentCache	content_cache = loadRelatedContent();
			
			enforceMaxResults( content_cache, reset_explicit );
		}
	}
	
	protected void
	enforceMaxResults(
		ContentCache		content_cache,
		boolean				reset_explicit )	
	{
		Map<String,DownloadInfo>		related_content			= content_cache.related_content;

		int num_to_remove = related_content.size() - ( max_results + temporary_space.get());
		
		if ( num_to_remove > 0 ){
			
			List<DownloadInfo>	infos = new ArrayList<DownloadInfo>(related_content.values());
				
			if ( reset_explicit ){
				
				for ( DownloadInfo info: infos ){
					
					if ( info.isExplicit()){
						
						info.setExplicit( false );
					}
				}
			}
			
			Collections.sort(
				infos,
				new Comparator<DownloadInfo>()
				{
					public int 
					compare(
						DownloadInfo o1, 
						DownloadInfo o2) 
					{
						int res = o2.getLevel() - o1.getLevel();
						
						if ( res != 0 ){
							
							return( res );
						}
						
						res = o1.getRank() - o2.getRank();
						
						if ( res != 0 ){
							
							return( res );
						}
						
						return( o1.getLastSeenSecs() - o2.getLastSeenSecs());
					}
				});

			List<RelatedContent> to_remove = new ArrayList<RelatedContent>();
			
			for (int i=0;i<Math.min( num_to_remove, infos.size());i++ ){
				
				to_remove.add( infos.get(i));
			}
			
			if ( to_remove.size() > 0 ){
					
				delete( to_remove.toArray( new RelatedContent[to_remove.size()]), content_cache, false );
			}
		}
	}
	
	public void
	addListener(
		RelatedContentManagerListener		listener )
	{
		listeners.add( listener );
	}
	
	public void
	removeListener(
		RelatedContentManagerListener		listener )
	{
		listeners.remove( listener );
	}
	
	protected static class
	ByteArrayHashMapEx<T>
		extends ByteArrayHashMap<T>
	{
	    public T
	    getRandomValueExcluding(
	    	T	excluded )
	    {
	    	int	num = RandomUtils.nextInt( size );
	    	
	    	T result = null;
	    	
	        for (int j = 0; j < table.length; j++) {
	        	
		         Entry<T> e = table[j];
		         
		         while( e != null ){
		        	 
	              	T value = e.value;
	               	
	              	if ( value != excluded ){
	              		
	              		result = value;
	              	}
	              	
	              	if ( num <= 0 && result != null ){
	              		
	              		return( result );
	              	}
	              	
	              	num--;
	              	
	              	e = e.next;
		        }
		    }
	    
	        return( result );
	    }
	}
	
	private Map<String,Object>
	serialiseDI(
		DownloadInfo			info,
		ContentCache			cc )
	{
		try{
			Map<String,Object> info_map = new HashMap<String,Object>();
			
			info_map.put( "h", info.getHash());
			
			ImportExportUtils.exportString( info_map, "d", info.getTitle());
			ImportExportUtils.exportInt( info_map, "r", info.getRand());
			ImportExportUtils.exportString( info_map, "t", info.getTracker());
			ImportExportUtils.exportLong( info_map, "z", info.getSize());
			
			ImportExportUtils.exportInt( info_map, "p", (int)( info.getPublishDate()/(60*60*1000)));
			ImportExportUtils.exportInt( info_map, "q", (info.getSeeds()<<16)|(info.getLeechers()&0xffff));
			ImportExportUtils.exportInt( info_map, "c", (int)info.getContentNetwork());

			byte[] tracker_keys = info.getTrackerKeys();
			if ( tracker_keys != null ){
				info_map.put( "k", tracker_keys );
			}
			
			byte[] ws_keys = info.getWebSeedKeys();
			if ( ws_keys != null ){
				info_map.put( "w", ws_keys );
			}
			
			String[] tags = info.getTags();
			if ( tags != null ){
				info_map.put( "g", encodeTags(tags));
			}
			
			byte nets = info.getNetworksInternal();
			if (nets != NET_PUBLIC ){
				info_map.put( "o", new Long(nets&0x00ff ));
			}
			
			if ( cc != null ){
							
				ImportExportUtils.exportBoolean( info_map, "u", info.isUnread());
				ImportExportUtils.exportIntArray( info_map, "l", info.getRandList());
				ImportExportUtils.exportInt( info_map, "s", info.getLastSeenSecs());
				ImportExportUtils.exportInt( info_map, "e", info.getLevel());
			}
			
			ImportExportUtils.exportLong(info_map, "cl", info.getChangedLocallyOn());
			
			return( info_map );
			
		}catch( Throwable e ){
			
			Debug.out( e );
			
			return( null );
		}
	}
	
	private DownloadInfo
	deserialiseDI(
		Map<String,Object>		info_map,
		ContentCache			cc )
	{
		try{
			byte[]	hash 	= (byte[])info_map.get("h");
			String	title	= ImportExportUtils.importString( info_map, "d" );
			int		rand	= ImportExportUtils.importInt( info_map, "r" );
			String	tracker	= ImportExportUtils.importString( info_map, "t" );
			long	size	= ImportExportUtils.importLong( info_map, "z" );
			
			int		date 			=  ImportExportUtils.importInt( info_map, "p", 0 );
			int		seeds_leechers 	=  ImportExportUtils.importInt( info_map, "q", -1 );
			byte	cnet 			=  (byte)ImportExportUtils.importInt( info_map, "c", (int)ContentNetwork.CONTENT_NETWORK_UNKNOWN );
			byte[]	tracker_keys	= (byte[])info_map.get( "k");
			byte[]	ws_keys			= (byte[])info_map.get( "w" );
			long lastChangedLocally = ImportExportUtils.importLong( info_map, "cl" );
			
			if ( tracker_keys != null && tracker_keys.length % 4 != 0 ){
				
				tracker_keys = null;
			}
		
			if ( ws_keys != null && ws_keys.length % 4 != 0 ){
				
				ws_keys = null;
			}
			
			byte[]	_tags	= (byte[])info_map.get( "g" );

			String[] tags = decodeTags( _tags );
			
			Long _nets = (Long)info_map.get( "o" );
			
			byte nets = _nets==null?NET_PUBLIC:_nets.byteValue();

			if ( cc == null ){
			
				 DownloadInfo info = new DownloadInfo( hash, hash, title, rand, tracker, tracker_keys, ws_keys, tags, nets, 0, false, size, date, seeds_leechers, cnet );
				 
				 info.setChangedLocallyOn( lastChangedLocally );
				 
				 return( info );
				
			}else{
				
				boolean unread = ImportExportUtils.importBoolean( info_map, "u" );
				
				int[] rand_list = ImportExportUtils.importIntArray( info_map, "l" );
				
				int	last_seen = ImportExportUtils.importInt( info_map, "s" );
				
				int	level = ImportExportUtils.importInt( info_map, "e" );
				
				DownloadInfo info = new DownloadInfo( hash, title, rand, tracker, tracker_keys, ws_keys, tags, nets, unread, rand_list, last_seen, level, size, date, seeds_leechers, cnet, cc );
				
				info.setChangedLocallyOn( lastChangedLocally );
        
				return( info );
			}
		}catch( Throwable e ){
			
			Debug.out( e );
			
			return( null );
		}
	}
	
	private void
	dump()
	{
		RelatedContent[] related_content = getRelatedContent();
		
		ByteArrayHashMap<List<String>>	tk_map = new ByteArrayHashMap<List<String>>();
		ByteArrayHashMap<List<String>>	ws_map = new ByteArrayHashMap<List<String>>();
		
		for ( RelatedContent rc: related_content ){
						
			byte[] tracker_keys = rc.getTrackerKeys();
			
			if ( tracker_keys != null ){
				
				for (int i=0;i<tracker_keys.length;i+=4 ){
					
					byte[] tk = new byte[4];
					
					System.arraycopy( tracker_keys, i, tk, 0, 4 );
					
					List<String> titles = tk_map.get( tk );
					
					if ( titles == null ){
						
						titles = new ArrayList<String>();
						
						tk_map.put( tk, titles );
					}
					
					titles.add( rc.getTitle());
				}
			}
			byte[] ws_keys = rc.getWebSeedKeys();
			
			if ( ws_keys != null ){
				
				for (int i=0;i<ws_keys.length;i+=4 ){
					
					byte[] wk = new byte[4];
					
					System.arraycopy( ws_keys, i, wk, 0, 4 );
					
					List<String> titles = ws_map.get( wk );
					
					if ( titles == null ){
						
						titles = new ArrayList<String>();
						
						ws_map.put( wk, titles );
					}
					
					titles.add( rc.getTitle());
				}
			}
		}
		
		System.out.println( "-- Trackers --" );
		
		for ( byte[] key: tk_map.keys()){
			
			List<String>	titles = tk_map.get( key );
			
			System.out.println( ByteFormatter.encodeString( key ));
			
			for ( String title: titles ){
				
				System.out.println( "    " + title );
			}
		}
		
		System.out.println( "-- Web Seeds --" );
		
		for ( byte[] key: ws_map.keys()){
			
			List<String>	titles = ws_map.get( key );
			
			System.out.println( ByteFormatter.encodeString( key ));
			
			for ( String title: titles ){
				
				System.out.println( "    " + title );
			}
		}
	}
	
	protected class
	DownloadInfo
		extends RelatedContent
	{
		final private int		rand;
		
		private boolean			unread	= true;
		private int[]			rand_list;
		private int				last_seen;
		private int				level;
		private boolean			explicit;
		
			// we *need* this reference here to manage garbage collection correctly
		
		private ContentCache	cc;
		
		protected
		DownloadInfo(
			byte[]		_related_to,
			byte[]		_hash,
			String		_title,
			int			_rand,
			String		_tracker,
			byte[]		_tracker_keys,
			byte[]		_ws_keys,
			String[]	_tags,
			byte		_nets,
			int			_level,
			boolean		_explicit,
			long		_size,
			int			_date,
			int			_seeds_leechers,
			byte		_cnet )
		{
			super( _related_to, _title, _hash, _tracker, _tracker_keys, _ws_keys, _tags, _nets, _size, _date, _seeds_leechers, _cnet );
			
			rand		= _rand;
			level		= _level;
			explicit	= _explicit;
			
			updateLastSeen();
		}
		
		protected
		DownloadInfo(
			byte[]			_hash,
			String			_title,
			int				_rand,
			String			_tracker,
			byte[]			_tracker_keys,
			byte[]			_ws_keys,
			String[]		_tags,
			byte			_nets,
			boolean			_unread,
			int[]			_rand_list,
			int				_last_seen,
			int				_level,
			long			_size,
			int				_date,
			int				_seeds_leechers,
			byte			_cnet,
			ContentCache	_cc )
		{
			super( _title, _hash, _tracker, _tracker_keys, _ws_keys, _tags, _nets, _size, _date, _seeds_leechers, _cnet );
			
			rand		= _rand;
			unread		= _unread;
			rand_list	= _rand_list;
			last_seen	= _last_seen;
			level		= _level;
			cc			= _cc;
			
			if ( rand_list != null ){
				
				if ( rand_list.length > MAX_RANK ){
					
					int[] temp = new int[ MAX_RANK ];
					
					System.arraycopy( rand_list, 0, temp, 0, MAX_RANK );
						
					rand_list = temp;
				}
			}
		}
		
		protected boolean
		addInfo(
			DownloadInfo		info )
		{
			boolean	result = false;
			
			synchronized( this ){
		
				updateLastSeen();
				
				int r = info.getRand();
				
				if ( rand_list == null ){
					
					rand_list = new int[]{ r };
										
					result	= true;
					
				}else{
					
					boolean	match = false;
					
					for (int i=0;i<rand_list.length;i++){
						
						if ( rand_list[i] == r ){
							
							match = true;
							
							break;
						}
					}
					
					if ( !match && rand_list.length < MAX_RANK ){
						
						int	len = rand_list.length;
						
						int[]	new_rand_list = new int[len+1];
						
						System.arraycopy( rand_list, 0, new_rand_list, 0, len );
						
						new_rand_list[len] = r;
						
						rand_list = new_rand_list;
						
						result = true;
					}
				}
			}
				
			if ( info.getLevel() < level ){
				
				level = info.getLevel();
				
				result = true;
			}
			
			long cn =  info.getContentNetwork();
			
			if ( 	cn != ContentNetwork.CONTENT_NETWORK_UNKNOWN && 
					getContentNetwork() == ContentNetwork.CONTENT_NETWORK_UNKNOWN ){
				
				setContentNetwork( cn );
			}
			
			int sl = info.getSeedsLeechers();
			
			if ( sl != -1 && sl != getSeedsLeechers()){
				
				setSeedsLeechers( sl );
				
				result = true;
			}
			
			int	d = info.getDateHours();
			
			if ( d > 0 && getDateHours() == 0 ){
				
				setDateHours( d );
				
				result = true;
			}
				
			String[] other_tags = info.getTags();
			
			if ( other_tags != null && other_tags.length > 0 ){
			
				String[] existing_tags = getTags();
				
				if ( existing_tags == NO_TAGS ){
					
					setTags( other_tags );
					
					result = true;
					
				}else{
					
					boolean	same;
					
					if ( other_tags.length == existing_tags.length ){
					
						if ( existing_tags.length == 1 ){
							
							same = other_tags[0].equals( existing_tags[0] );
							
						}else{
							
							same = true;
							
							for ( int i=0;i<existing_tags.length;i++ ){
								
								String e_tag = existing_tags[i];
								
								boolean	found = false;
								
								for ( int j=0;j<other_tags.length;j++){
									
									if ( e_tag.equals( other_tags[j])){
										
										found = true;
										
										break;
									}
								}
								
								if ( !found ){
									
									same = false;
									
									break;
								}
							}
						}
					}else{
					
						same = false;
					}
					
					if ( !same ){
						
						Set<String>	tags = new HashSet<String>();
						
						for ( String t: existing_tags ){
							tags.add( t );
						}
						for ( String t: other_tags ){
							tags.add( t );
						}
						
						setTags( tags.toArray( new String[tags.size()]));
						
						result = true;
					}
				}
			}
			
			byte	other_nets 		= info.getNetworksInternal();
			byte	existing_nets	= getNetworksInternal();
			
			if ( other_nets != existing_nets ){
				
				setNetworksInternal((byte)( other_nets | existing_nets ));
				
				result = true;
			}
			
			if (result) {
				setChangedLocallyOn(0);
			}
			return( result );
		}
		
		public int
		getLevel()
		{
			return( level );
		}
		
		protected boolean
		isExplicit()
		{
			return( explicit );
		}
		
		protected void
		setExplicit(
			boolean		b )
		{
			explicit	= b;
		}
		
		protected void
		updateLastSeen()
		{
				// persistence of this is piggy-backed on other saves to limit resource usage
				// only therefore a vague measure

			last_seen	= (int)( SystemTime.getCurrentTime()/1000 );
		}
		
		public int
		getRank()
		{
			return( rand_list==null?0:rand_list.length );
		}
		
		public boolean
		isUnread()
		{
			return( unread );
		}
		
		protected void
		setPublic(
			ContentCache	_cc )
		{
			cc	= _cc;
			
			if ( unread ){
				
				incrementUnread();
			}
			
			rand_list = new int[]{ rand };
			setChangedLocallyOn(0);
		}
		
		public int
		getLastSeenSecs()
		{
			return( last_seen );
		}
		
		protected void
		setUnreadInternal(
			boolean	_unread )
		{
			synchronized( this ){

				unread = _unread;
			}
		}
		
		public void
		setUnread(
			boolean	_unread )
		{
			boolean	changed = false;
			
			synchronized( this ){
				
				if ( unread != _unread ){
				
					unread = _unread;
					
					changed = true;
				}
			}
			
			if ( changed ){
			
				if ( _unread ){
					
					incrementUnread();
					
				}else{
					
					decrementUnread();
				}
				
				setChangedLocallyOn(0);
				contentChanged( this );
			}
		}
		
		protected int
		getRand()
		{
			return( rand );
		}
		
		protected int[]
		getRandList()
		{
			return( rand_list );
		}
		
		public Download 
		getRelatedToDownload() 
		{
			try{
				return( getDownload( getRelatedToHash()));
				
			}catch( Throwable e ){
				
				Debug.out( e );
				
				return( null );
			}
		}
		
		public void 
		delete() 
		{
			setChangedLocallyOn(0);
			RelatedContentManager.this.delete( new RelatedContent[]{ this });
		}
		
		public String
		getString()
		{
			return( super.getString() + ", " + rand + ", rl=" + rand_list + ", last_seen=" + last_seen + ", level=" + level );
		}
	}
	
	// can't move this class out of here as the key for the transfer type is based on its
	// class name...
	
	protected RCMSearchXFer			transfer_type = new RCMSearchXFer();

	protected class
	RCMSearchXFer
		implements DistributedDatabaseTransferType
	{	
	}
	
	protected static class
	ContentCache
	{
		protected Map<String,DownloadInfo>						related_content			= new HashMap<String, DownloadInfo>();
		protected ByteArrayHashMapEx<ArrayList<DownloadInfo>>	related_content_map		= new ByteArrayHashMapEx<ArrayList<DownloadInfo>>();
	}
	
	private static class
	SecondaryLookup
	{
		final private byte[]	hash;
		final private int		level;
		final private byte		nets;
		
		private
		SecondaryLookup(
			byte[]		_hash,
			int			_level,
			byte		_nets )
		{
			hash	= _hash;
			level	= _level;
			nets	= _nets;
		}
		
		private byte[]
		getHash()
		{
			return( hash );
		}
		
		private int
		getLevel()
		{
			return( level );
		}
		
		private byte
		getNetworks()
		{
			return( nets );
		}
	}
}
