/*
 * Created on Mar 21, 2006 3:09:00 PM
 * Copyright (C) Azureus Software, Inc, All Rights Reserved.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 */
package org.gudy.azureus2.core3.util;

import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.bouncycastle.util.encoders.Base64;
import org.gudy.azureus2.core3.config.COConfigurationManager;
import org.gudy.azureus2.core3.download.DownloadManager;
import org.gudy.azureus2.core3.torrent.TOTorrent;
import org.gudy.azureus2.plugins.download.Download;
import org.gudy.azureus2.plugins.torrent.Torrent;
import org.gudy.azureus2.plugins.torrent.TorrentAnnounceURLList;
import org.gudy.azureus2.plugins.torrent.TorrentAnnounceURLListSet;
import org.gudy.azureus2.plugins.utils.resourcedownloader.ResourceDownloader;
import org.gudy.azureus2.plugins.utils.resourceuploader.ResourceUploader;
import org.gudy.azureus2.pluginsimpl.local.PluginCoreUtils;

import com.aelitis.net.magneturi.MagnetURIHandler;

/**
 * @author TuxPaper
 * @created Mar 21, 2006
 *
 */
public class UrlUtils
{
	private static final ThreadPool	connect_pool = new ThreadPool( "URLConnectWithTimeout", 8, true );
	
	static{
		connect_pool.setWarnWhenFull();
	}
	
	private static final String[] prefixes = new String[] {
			"http://",
			"https://",
			"ftp://",
			"dht://",
			"magnet:?",
			"magnet://?",
			"maggot://" };

	private static int MAGNETURL_STARTS_AT = 3;	// dht:// is a form of magnet URL
	
	private static final Object[] XMLescapes = new Object[] {
		new String[] { "&", "&amp;" },
		new String[] { ">", "&gt;" },
		new String[] { "<", "&lt;" },
		new String[] { "\"", "&quot;" },
		new String[] { "'", "&apos;" },
	};

	public static String
	getMagnetURI(
		byte[]		hash )
	{
		return( "magnet:?xt=urn:btih:" + Base32.encode( hash ));
	}

	public static String
	getMagnetURI(
		byte[]		hash,
		String		name,
		String[]	networks )
	{
		String magnet_uri = getMagnetURI( hash );
					
		magnet_uri += encodeName( name );
					
		magnet_uri += encodeNetworks( networks );
		
		return( magnet_uri );
	}

	private static String
	encodeName(
		String	name )
	{
		if ( name == null ){
			
			return( "" );
			
		}else{
			
			return( "&dn=" + UrlUtils.encode(name));
		}
	}
	
	private static String
	encodeNetworks(
		String[]	networks )
	{
		String	net_str = "";

		if ( networks != null && networks.length > 0 ){
						
			for ( String net: networks ){
				
				if ( net == AENetworkClassifier.AT_PUBLIC && networks.length == 1 ){
						
					break;
				}
				
				net_str += "&net=" + net;
			}
		}
		
		return( net_str );
	}
	
	public static Set<String>
	extractNetworks(
		String[]		magnet_uri )
	{
		String magnet_uri_in = magnet_uri[0];
		
		Set<String>	result = new HashSet<String>();
		
		int	pos = magnet_uri_in.indexOf( '?' );
		
		if ( pos != -1 ){
		
			String magnet_uri_out = magnet_uri_in.substring( 0, pos+1 );

			String[] bits = magnet_uri_in.substring( pos+1 ).split( "&" );
			
			for ( String bit: bits ){
				
				String[] temp = bit.split( "=", 2 );
				
				boolean	remove = false;
				
				if ( temp.length == 2 ){
					
					String	lhs = temp[0];
					
					if ( lhs.equalsIgnoreCase( "net" )){
					
						String	rhs = decode( temp[1] );
					
						result.add( AENetworkClassifier.internalise( rhs ));
						
						remove = true;
					}
				}
				
				if ( !remove ){
					
					if ( !magnet_uri_out.endsWith( "?" )){
						
						magnet_uri_out += "&";
					}
					
					magnet_uri_out += bit;		
				}
			}
			
			if ( result.size() > 0 ){
			
				magnet_uri[0] = magnet_uri_out;
			}
		}
		
		return( result );
	}
	
	public static String
	getMagnetURI(
		Download		download )
	{
		return( getMagnetURI( PluginCoreUtils.unwrap(download)));
	}
	
	public static String
	getMagnetURI(
		Download		download,
		int				max_name_len )
	{
		return( getMagnetURI( PluginCoreUtils.unwrap(download), max_name_len ));
	}
	
	public static String
	getMagnetURI(
		DownloadManager		dm )
	{
		return( getMagnetURI( dm, Integer.MAX_VALUE ));
	}
	
	public static String
	getMagnetURI(
		DownloadManager		dm,
		int					max_name_len )
	{
		if ( dm == null ){
			
			return( null );
		}
		
		TOTorrent to_torrent = dm.getTorrent();
		
		if ( to_torrent == null ){
		
			return( null );
		}
		
		String name = dm.getDisplayName();
		
		if ( name.length() > max_name_len ){
			
			name = name.substring( 0, max_name_len-3) + "...";
		}
		
		String magnet_uri = getMagnetURI( name, PluginCoreUtils.wrap( to_torrent ));
		
		String[]	networks = dm.getDownloadState().getNetworks();
		
		magnet_uri += encodeNetworks( networks );
		
		return( magnet_uri );
	}
	
	public static String
	getMagnetURI(
		String		name,
		Torrent		torrent )
	{
		String	magnet_str = getMagnetURI( torrent.getHash());
		
		magnet_str += encodeName( name);

		List<String>	tracker_urls = new ArrayList<String>();
		
		URL announce_url = torrent.getAnnounceURL();
		
		if ( announce_url != null ){
			
			if ( !TorrentUtils.isDecentralised( announce_url )){
				
				tracker_urls.add( announce_url.toExternalForm());
			}
		}
		
		TorrentAnnounceURLList list = torrent.getAnnounceURLList();
		
		TorrentAnnounceURLListSet[] sets = list.getSets();
		
		for ( TorrentAnnounceURLListSet set: sets ){
			
			URL[] set_urls = set.getURLs();
			
			if ( set_urls.length > 0 ){
				
				URL set_url = set_urls[0];
				
				if ( !TorrentUtils.isDecentralised( set_url )){
					
					String str = set_url.toExternalForm();
					
					if ( !tracker_urls.contains( str )){
					
						tracker_urls.add( str );
					}
				}
			}
		}
		
		for ( String str: tracker_urls ){
			
			magnet_str += "&tr=" + UrlUtils.encode( str );
		}
		
		List<String>	ws_urls = new ArrayList<String>();

		Object obj = torrent.getAdditionalProperty( "url-list" );
							
		if ( obj instanceof byte[] ){
            
			try{
				ws_urls.add( new URL( new String((byte[])obj, "UTF-8" )).toExternalForm());
				
			}catch( Throwable e ){							
			}
		}else if ( obj instanceof List ){
			
			for ( Object o: (List)obj ){
				
				try{
					if (o instanceof byte[]) {
						ws_urls.add( new URL( new String((byte[])o, "UTF-8" )).toExternalForm());
					} else if (o instanceof String) {
						ws_urls.add( new URL((String) o).toExternalForm());
					}
					
				}catch( Throwable e ){							
				}
			}
		} else if ( obj instanceof String ) {
			try{
				ws_urls.add(new URL((String) obj).toExternalForm());
			}catch( Throwable e ){							
			}
		}
		
		for ( String str: ws_urls ){
			
			magnet_str += "&ws=" + UrlUtils.encode( str );
		}	
		
		return( magnet_str );
	}
		/**
		 * returns magnet uri if input is base 32 or base 16 encoded sha1 hash, null otherwise
		 * @param base_hash
		 * @return
		 */
	
	public static String
	normaliseMagnetURI(
		String		base_hash )
	{
		byte[]	hash = decodeSHA1Hash( base_hash );
		
		if ( hash != null ){
			
			return( getMagnetURI( hash ));
		}
		
		return( null );
	}
	
	public static byte[]
	decodeSHA1Hash(
		String	str )
	{
		if ( str == null ){
			
			return( null );
		}
		
		str = str.trim();
		
		byte[] hash = null;
		
		try{
			if ( str.length() == 40 ){
				
				hash = ByteFormatter.decodeString( str );
				
			}else if ( str.length() == 32 ){
				
				hash = Base32.decode( str );
			}
		}catch( Throwable e ){
		}
		
		if ( hash != null ){
			
			if ( hash.length != 20 ){
								
				hash = null;
			}
		}
		
		return( hash );
	}
	
	/**
	 * test string for possibility that it's an URL.  Considers 40 byte hex 
	 * strings as URLs
	 * 
	 * @param sURL
	 * @return
	 */
	public static boolean isURL(String sURL) {
		return parseTextForURL(sURL, true) != null;
	}

	public static boolean isURL(String sURL, boolean bGuess) {
		return parseTextForURL(sURL, true, bGuess) != null;
	}

	public static String parseTextForURL(String text, boolean accept_magnets) {
		return parseTextForURL(text, accept_magnets, true);
	}

	public static String
	getURL(
		String	text )
	{
		return( parseTextForURL(text, false, false ));
	}
	
	public static String parseTextForURL(String text, boolean accept_magnets,
			boolean guess) {

		if (text == null || text.length() < 5) {
			return null;
		}

		text = text.trim();
		
		if ( text.startsWith( "azplug:" )){
			
			return( text );
		}
		
		if ( text.startsWith( "chat:" )){
			
			return( "azplug:?id=azbuddy&arg=" + UrlUtils.encode( text ));
		}
		
		String href = parseHTMLforURL(text);
		if (href != null) {
			return href;
		}

		try {
			text = text.trim();
			text = decodeIfNeeded(text);
		} catch (Exception e) {
			// sometimes fires a IllegalArgumentException
			// catch everything and ignore.
		}
		
		String textLower;
		try {
			textLower = text.toLowerCase();
		} catch (Throwable e) {
			textLower = text;
		}
		int max = accept_magnets ? prefixes.length : MAGNETURL_STARTS_AT;
		int end = -1;
		int start = textLower.length();
		String strURL = null;
		for (int i = 0; i < max; i++) {
			final int testBegin = textLower.indexOf(prefixes[i]);
			if (testBegin >= 0 && testBegin < start) {
				end = text.indexOf("\n", testBegin + prefixes[i].length());
				String strURLTest = (end >= 0) ? text.substring(testBegin, end - 1)
						: text.substring(testBegin);
				try {
					URL parsedURL = new URL(strURLTest);
					strURL = parsedURL.toExternalForm();
				} catch (MalformedURLException e1) {
					e1.printStackTrace();
					if (i >= MAGNETURL_STARTS_AT) {
						strURL = strURLTest;
					}
				}
			}
		}
		if (strURL != null) {
			return strURL;
		}
		
		if (new File(text).exists()) {
			return null;
		}

		if (accept_magnets
				&& (text.startsWith("bc://") || text.startsWith("bctp://"))) {
			return parseTextForMagnets(text);
		}

		// accept raw hash of 40 hex chars
		if (accept_magnets && text.matches("^[a-fA-F0-9]{40}$")) {
			// convert from HEX to raw bytes
			byte[] infohash = ByteFormatter.decodeString(text.toUpperCase());
			// convert to BASE32
			return "magnet:?xt=urn:btih:" + Base32.encode(infohash);
		}

		// accept raw hash of 32 base-32 chars
		if (accept_magnets && text.matches("^[a-zA-Z2-7]{32}$")) {
			return "magnet:?xt=urn:btih:" + text;
		}
		
		// javascript:loadOrAlert('WVOPRHRPFSCLAW7UWHCXCH7QNQIU6TWG')

		// accept raw hash of 32 base-32 chars, with garbage around it
		if (accept_magnets && guess) {
			Pattern pattern = Pattern.compile("[^a-zA-Z2-7][a-zA-Z2-7]{32}[^a-zA-Z2-7]");
			Matcher matcher = pattern.matcher(text);
			if (matcher.find()) {
				String hash = text.substring(matcher.start() + 1, matcher.start() + 33);
				return "magnet:?xt=urn:btih:" + hash;
			}

			pattern = Pattern.compile("[^a-fA-F0-9][a-fA-F0-9]{40}[^a-fA-F0-9]");
			matcher = pattern.matcher(text);
			if (matcher.find()) {
				String hash = text.substring(matcher.start() + 1, matcher.start() + 41);
				// convert from HEX to raw bytes
				byte[] infohash = ByteFormatter.decodeString(hash.toUpperCase());
				// convert to BASE32
				return "magnet:?xt=urn:btih:" + Base32.encode(infohash);
			}
		}
		
		return null;
	}

	public static String
	parseTextForMagnets(
		String		text )
	{
		if (text.startsWith("magnet:") || text.startsWith( "maggot:" )){
			return text;
		}

		// accept raw hash of 40 hex chars
		if (text.matches("^[a-fA-F0-9]{40}$")) {
			// convert from HEX to raw bytes
			byte[] infohash = ByteFormatter.decodeString(text.toUpperCase());
			// convert to BASE32
			return "magnet:?xt=urn:btih:" + Base32.encode(infohash);
		}

		// accept raw hash of 32 base-32 chars
		if (text.matches("^[a-zA-Z2-7]{32}$")) {
			return "magnet:?xt=urn:btih:" + text;
		}
		
		Pattern pattern;
		Matcher matcher;
		
		pattern = Pattern.compile("magnet:\\?[a-z%0-9=_:&.]+", Pattern.CASE_INSENSITIVE);
		matcher = pattern.matcher(text);
		if (matcher.find()) {
			return matcher.group();
		}

		pattern = Pattern.compile("maggot://[a-z0-9]+:[a-z0-9]", Pattern.CASE_INSENSITIVE);
		matcher = pattern.matcher(text);
		if (matcher.find()) {
			return matcher.group();
		}
		
		pattern = Pattern.compile("bc://bt/([a-z0-9=\\+/]+)", Pattern.CASE_INSENSITIVE);
		matcher = pattern.matcher(text.replaceAll(" ", "+"));
		if (matcher.find()) {
			String base64 = matcher.group(1);
			byte[] decode = Base64.decode(base64);
			if (decode != null && decode.length > 0) {
				// Format is AA/<name>/<size>/<hash>/ZZ
				try {
					String decodeString = new String(decode, "utf8");
					pattern = Pattern.compile("AA.*/(.*)/ZZ", Pattern.CASE_INSENSITIVE);
					matcher = pattern.matcher(decodeString);
					if (matcher.find()) {
						String hash = matcher.group(1);
						String magnet = parseTextForMagnets(hash);
						if (magnet != null) {
							pattern = Pattern.compile("AA/(.*)/[0-9]+", Pattern.CASE_INSENSITIVE);
							matcher = pattern.matcher(decodeString);
							if (matcher.find()) {
								String name = matcher.group(1);
								return magnet + "&dn=" + encode(name);
							}
							return magnet;
						}
					}
				} catch (UnsupportedEncodingException e) {
				}
			}
		}

		pattern = Pattern.compile("bctp://task/(.*)", Pattern.CASE_INSENSITIVE);
		matcher = pattern.matcher(text);
		if (matcher.find()) {
			// Format is <name>/<size>/<hash>
			String decodeString = matcher.group(1);
			String magnet = parseTextForMagnets(decodeString);
			if (magnet != null) {
				pattern = Pattern.compile("(.*)/[0-9]+", Pattern.CASE_INSENSITIVE);
				matcher = pattern.matcher(decodeString);
				if (matcher.find()) {
					String name = matcher.group(1);
					return magnet + "&dn=" + encode(name);
				}
				return magnet;
			}
		}

		// accept raw hash of 32 base-32 chars, with garbage around it
		if (true) {
			text = "!" + text + "!";
			pattern = Pattern.compile("[^a-zA-Z2-7][a-zA-Z2-7]{32}[^a-zA-Z2-7]");
			matcher = pattern.matcher(text);
			if (matcher.find()) {
				String hash = text.substring(matcher.start() + 1, matcher.start() + 33);
				return "magnet:?xt=urn:btih:" + hash;
			}

			pattern = Pattern.compile("[^a-fA-F0-9][a-fA-F0-9]{40}[^a-fA-F0-9]");
			matcher = pattern.matcher(text);
			if (matcher.find()) {
				String hash = text.substring(matcher.start() + 1, matcher.start() + 41);
				// convert from HEX to raw bytes
				byte[] infohash = ByteFormatter.decodeString(hash.toUpperCase());
				// convert to BASE32
				return "magnet:?xt=urn:btih:" + Base32.encode(infohash);
			}
		}
		
		return( null );
	}
	
	private static String parseHTMLforURL(String text) {
		if (text == null) {
			return null;
		}

		// examples:
		// <A HREF=http://abc.om/moo>test</a>
		// <A style=cow HREF="http://abc.om/moo">test</a>
		// <a href="http://www.gnu.org/licenses/fdl.html" target="_top">moo</a>

		Pattern pat = Pattern.compile("<.*a\\s++.*href=\"?([^\\'\"\\s>]++).*",
				Pattern.CASE_INSENSITIVE);
		Matcher m = pat.matcher(text);
		if (m.find()) {
			String sURL = m.group(1);
			try {
				sURL = decodeIfNeeded(sURL);
			} catch (Exception e) {
				// sometimes fires a IllegalArgumentException
				// catch everything and ignore.
			}
			return sURL;
		}

		return null;
	}



	/**
	 * Like URLEncoder.encode, except translates spaces into %20 instead of +
	 * @param s
	 * @return
	 */
	public static String encode(String s) {
		if (s == null) {
			return "";
		}
		try {
			return URLEncoder.encode(s, "UTF-8").replaceAll("\\+", "%20");
		} catch (UnsupportedEncodingException e) {
			return URLEncoder.encode(s).replaceAll("\\+", "%20");
		}
	}
	
	public static String decode(String s) {
		if (s == null) {
			return "";
		}
		try {
			try{
				return( URLDecoder.decode(s, "UTF-8"));
				
			}catch( IllegalArgumentException e ){
				
					// handle truncated encodings somewhat gracefully
				
				int pos = s.lastIndexOf( "%" );
				
				if ( pos >= s.length()-2 ){
					
					return( URLDecoder.decode(s.substring( 0, pos ), "UTF-8"));
				}
				
				throw( e );
			}
		} catch (UnsupportedEncodingException e) {
			
			return( URLDecoder.decode(s));
		}
	}
	
	/**
	 * Unfortunately we have code that mindlessly decoded URLs (FileDownloadWindow) which borked (in the case I discovered) magnet uris with encoded 
	 * parameters (e.g. the &tr= parameter) - doing so screws stuff up later if, for example, the parameter included an encoded '&'
	 * @param s
	 * @return
	 */
	
	public static String 
	decodeIfNeeded(
		String s ) 
	{
		if ( s == null ){
			
			return( "" );
		}
		
		try{
				// of course someone prolly added the blind URLDecode for a reason so try and just deal with the borkage
				// which means looking for &a=b elements and ensure we don't URLDecode the 'b'
			
				// only have issues if there's a naked '?' there
			
			int	q_pos = s.indexOf( '?' );
			int a_pos = s.indexOf( '&' );
			
			if ( q_pos == -1 && a_pos == -1 ){
				
				return( decode( s ));
			}
			
			int start = Math.min( q_pos, a_pos );
			
			return( decode( s.substring( 0, start )) + s.substring( start ));
			
		}catch( Throwable e ){
			
			return( s );
		}
	}
	
	public static String escapeXML(String s) {
		if (s == null) {
			return "";
		}
		String ret = s;
		for (int i = 0; i < XMLescapes.length; i++) {
			String[] escapeEntry = (String[])XMLescapes[i];
			ret = ret.replaceAll(escapeEntry[0], escapeEntry[1]);
		}
		return ret;
	}

	public static String unescapeXML(String s) {
		if (s == null) {
			return "";
		}
		String ret = s;
		for (int i = 0; i < XMLescapes.length; i++) {
			String[] escapeEntry = (String[])XMLescapes[i];
			ret = ret.replaceAll(escapeEntry[1], escapeEntry[0]);
		}
		return ret;
	}
	
	public static String
	convertIPV6Host(
		String	host )
	{
		if ( host.indexOf(':') != -1 ){
			
			return( "[" + host + "]" );
		}
		
		return( host );
	}
	
	public static String
	expandIPV6Host(
		String	host )
	{
		if ( host.indexOf(':') != -1 ){
			
			try{
				return( InetAddress.getByAddress(InetAddress.getByName( host ).getAddress()).getHostAddress());
				
			}catch( Throwable e ){
				
				Debug.printStackTrace(e);
			}
		}
		
		return( host );
	}
	
	public static void
	connectWithTimeout(
		final URLConnection		connection,
		long					connect_timeout )
	
		throws IOException
	{
		connectWithTimeouts( connection, connect_timeout, -1 );
	}
	
	public static void
	connectWithTimeouts(
		final URLConnection		connection,
		long					connect_timeout,
		long					read_timeout )
	
		throws IOException
	{
		if ( connect_timeout != -1 ){
				
			connection.setConnectTimeout( (int)connect_timeout );	
		}
			
		if ( read_timeout != -1 ){
				
			connection.setReadTimeout( (int)read_timeout );	
		}
			
		connection.connect();
	}
	
	private static String	last_headers = COConfigurationManager.getStringParameter( "metasearch.web.last.headers", null );
	
	// private static final String default_headers = "SG9zdDogbG9jYWxob3N0OjQ1MTAwClVzZXItQWdlbnQ6IE1vemlsbGEvNS4wIChXaW5kb3dzOyBVOyBXaW5kb3dzIE5UIDUuMTsgZW4tVVM7IHJ2OjEuOC4xLjE0KSBHZWNrby8yMDA4MDQwNCBGaXJlZm94LzIuMC4wLjE0CkFjY2VwdDogdGV4dC94bWwsYXBwbGljYXRpb24veG1sLGFwcGxpY2F0aW9uL3hodG1sK3htbCx0ZXh0L2h0bWw7cT0wLjksdGV4dC9wbGFpbjtxPTAuOCxpbWFnZS9wbmcsKi8qO3E9MC41CkFjY2VwdC1MYW5ndWFnZTogZW4tdXMsZW47cT0wLjUKQWNjZXB0LUVuY29kaW5nOiBnemlwLGRlZmxhdGUKQWNjZXB0LUNoYXJzZXQ6IElTTy04ODU5LTEsdXRmLTg7cT0wLjcsKjtxPTAuNwpLZWVwLUFsaXZlOiAzMDAKQ29ubmVjdGlvbjoga2VlcC1hbGl2ZQ==";
	private static final String default_headers = "QWNjZXB0OiB0ZXh0L2h0bWwsYXBwbGljYXRpb24veGh0bWwreG1sLGFwcGxpY2F0aW9uL3htbDtxPTAuOSwqLyo7cT0wLjgKQWNjZXB0LUNoYXJzZXQ6IElTTy04ODU5LTEsdXRmLTg7cT0wLjcsKjtxPTAuMwpBY2NlcHQtRW5jb2Rpbmc6IGd6aXAsZGVmbGF0ZQpBY2NlcHQtTGFuZ3VhZ2U6IGVuLVVTLGVuO3E9MC44CkNhY2hlLUNvbnRyb2w6IG1heC1hZ2U9MApDb25uZWN0aW9uOiBrZWVwLWFsaXZlClVzZXItQWdlbnQ6IE1vemlsbGEvNS4wIChXaW5kb3dzIE5UIDYuMTsgV09XNjQpIEFwcGxlV2ViS2l0LzUzNi4xMSAoS0hUTUwsIGxpa2UgR2Vja28pIENocm9tZS8yMC4wLjExMzIuNDcgU2FmYXJpLzUzNi4xMQ==";

	public static void
	setBrowserHeaders(
		ResourceDownloader		rd,
		String					referer )
	{
		setBrowserHeaders( rd, null, referer );
	}
	
	public static void
	setBrowserHeaders(
		ResourceDownloader		rd,
		String					encoded_headers,
		String					referer )
	{
		String	headers_to_use = getBrowserHeadersToUse( encoded_headers );
		
		try{
			String header_string = new String( Base64.decode( headers_to_use ), "UTF-8" );
		
			String[]	headers = header_string.split( "\n" );
			
			for (int i=0;i<headers.length;i++ ){
			
				String	header = headers[i];
				
				int	pos = header.indexOf( ':' );
				
				if ( pos != -1 ){
					
					String	lhs = header.substring(0,pos).trim();
					String	rhs	= header.substring(pos+1).trim();
					
					if ( !( lhs.equalsIgnoreCase( "Host") || 
							lhs.equalsIgnoreCase( "Referer" ))){
						
						rd.setProperty( "URL_" + lhs, rhs );
					}
				}
			}
			
			if ( referer != null && referer.length() > 0 ){
				
				rd.setProperty( "URL_Referer", referer );
			}
		}catch( Throwable e ){	
		}
	}
	
	public static void
	setBrowserHeaders(
		ResourceUploader		ru,
		String					encoded_headers,
		String					referer )
	{
		String	headers_to_use = getBrowserHeadersToUse( encoded_headers );
		
		try{
			String header_string = new String( Base64.decode( headers_to_use ), "UTF-8" );
		
			String[]	headers = header_string.split( "\n" );
			
			for (int i=0;i<headers.length;i++ ){
			
				String	header = headers[i];
				
				int	pos = header.indexOf( ':' );
				
				if ( pos != -1 ){
					
					String	lhs = header.substring(0,pos).trim();
					String	rhs	= header.substring(pos+1).trim();
					
					if ( !( lhs.equalsIgnoreCase( "Host") || 
							lhs.equalsIgnoreCase( "Referer" ))){
						
						ru.setProperty( "URL_" + lhs, rhs );
					}
				}
			}
			
			if ( referer != null && referer.length() > 0 ){
				
				ru.setProperty( "URL_Referer", referer );
			}
		}catch( Throwable e ){	
		}
	}
	
	public static void
	setBrowserHeaders(
		URLConnection			connection,
		String					referer )
	{
		setBrowserHeaders( connection, null, referer );
	}
	
	public static void
	setBrowserHeaders(
		URLConnection			connection,
		String					encoded_headers,
		String					referer )
	{
		String	headers_to_use = getBrowserHeadersToUse( encoded_headers );
		
		try{
		
			String header_string = new String( Base64.decode( headers_to_use ), "UTF-8" );
		
			String[]	headers = header_string.split( "\n" );
			
			for (int i=0;i<headers.length;i++ ){
			
				String	header = headers[i];
				
				int	pos = header.indexOf( ':' );
				
				if ( pos != -1 ){
					
					String	lhs = header.substring(0,pos).trim();
					String	rhs	= header.substring(pos+1).trim();
					
					if ( !( lhs.equalsIgnoreCase( "Host") || 
							lhs.equalsIgnoreCase( "Referer" ))){
						
						connection.setRequestProperty( lhs, rhs );
					}
				}
			}
			
			if ( referer != null && referer.length() > 0 ){
				
				connection.setRequestProperty( "Referer", referer );
			}
		}catch( Throwable e ){		
		}
	}
	
	public static Map
	getBrowserHeaders(
		String					referer )
	{
		String	headers_to_use = getBrowserHeadersToUse( null );
		
		Map	result = new HashMap();
		
		try{
		
			String header_string = new String( Base64.decode( headers_to_use ), "UTF-8" );
		
			String[]	headers = header_string.split( "\n" );
			
			for (int i=0;i<headers.length;i++ ){
			
				String	header = headers[i];
				
				int	pos = header.indexOf( ':' );
				
				if ( pos != -1 ){
					
					String	lhs = header.substring(0,pos).trim();
					String	rhs	= header.substring(pos+1).trim();
					
					if ( !( lhs.equalsIgnoreCase( "Host") || 
							lhs.equalsIgnoreCase( "Referer" ))){
						
						result.put( lhs, rhs );
					}
				}
			}
			
			if ( referer != null && referer.length() > 0){
				
				result.put( "Referer", referer );
			}
		}catch( Throwable e ){		
		}
		
		return( result );
	}
	
	private static String
	getBrowserHeadersToUse(
		String		encoded_headers )
	{
		String	headers_to_use = encoded_headers;
		
		synchronized( UrlUtils.class ){
			
			if ( headers_to_use == null ){
				
				if ( last_headers != null ){
					
					headers_to_use = last_headers;
					
				}else{
					
					headers_to_use = default_headers;
				}
			}else{
			
				if ( last_headers == null || !headers_to_use.equals( last_headers )){
					
					COConfigurationManager.setParameter( "metasearch.web.last.headers", headers_to_use );
				}
				
				last_headers = headers_to_use;
			}
		}
		
		return( headers_to_use );
	}
	
	public static boolean queryHasParameter(String query_string, String param_name, boolean case_sensitive) {
		if (!case_sensitive) {
			query_string = query_string.toLowerCase();
			param_name = param_name.toLowerCase();
		}
		if (query_string.charAt(0) == '?') {
			query_string = '&' + query_string.substring(1);
		}
		else if (query_string.charAt(0) != '&') {
			query_string = '&' + query_string;
		}
		
		return query_string.indexOf("&" + param_name + "=") != -1;
	}
	
	public static boolean 
	containsPasskey(
		URL		url )
	{
		if ( url == null ){
			
			return( false );
		}
		
		String url_str = url.toExternalForm();
		
		return( url_str.matches(".*[0-9a-z]{20,40}.*"));
	}
	
	public static URL
	setPort(
		URL		u,
		int		port )
	{
		if ( port == -1 ){
			port = u.getDefaultPort();
		}
		StringBuffer result = new StringBuffer();
		result.append(u.getProtocol());
		result.append(":");
		String authority=u.getAuthority();
		if (authority != null && authority.length() > 0) {
			result.append("//");
			int pos = authority.indexOf( '@' );
			if ( pos != -1 ){
				result.append(authority.substring(0,pos+1));
				authority = authority.substring(pos+1);
			}
			pos = authority.lastIndexOf(':');
			if ( pos == -1 ){
				if ( port > 0 ){
					result.append(authority + ":" + port );
				}else{
					result.append(authority);
				}
			}else{
				if ( port > 0 ){
					result.append(authority.substring(0,pos+1) + port );
				}else{
					result.append(authority.substring(0,pos));
				}
			}
		}
		if (u.getPath() != null) {
			result.append(u.getPath());
		}
		if (u.getQuery() != null) {
			result.append('?');
			result.append(u.getQuery());
		}
		if (u.getRef() != null) {
			result.append("#");
			result.append(u.getRef());
		}
		try{
			return( new URL( result.toString()));
		}catch( Throwable e ){
			Debug.out(e);
			return(u);
		}
	}
	
	public static URL
	setHost(
		URL			u,
		String		host )
	{
		StringBuffer result = new StringBuffer();
		result.append(u.getProtocol());
		result.append(":");
		String authority=u.getAuthority();
		if (authority != null && authority.length() > 0) {
			result.append("//");
			int pos = authority.indexOf( '@' );
			if ( pos != -1 ){
				result.append(authority.substring(0,pos+1));
				authority = authority.substring(pos+1);
			}
			pos = authority.lastIndexOf(':');
			if ( pos == -1 ){
				result.append(host );
			}else{
				result.append(host + authority.substring(pos));				
			}
		}
		if (u.getPath() != null) {
			result.append(u.getPath());
		}
		if (u.getQuery() != null) {
			result.append('?');
			result.append(u.getQuery());
		}
		if (u.getRef() != null) {
			result.append("#");
			result.append(u.getRef());
		}
		try{
			return( new URL( result.toString()));
		}catch( Throwable e ){
			Debug.out(e);
			return(u);
		}
	}
	
	public static URL
	setProtocol(
		URL			u,
		String		protocol )
	{
		String str = u.toExternalForm();
		
		int pos = str.indexOf( ":" );
		
		try{
			return( new URL( protocol + str.substring( pos )));
			
		}catch( Throwable e ){
			
			Debug.out( e );
			
			return( u );
		}
	}
	
	public static URL
	getBaseURL(
		URL		u )
	{
		StringBuffer result = new StringBuffer();
		result.append(u.getProtocol());
		result.append(":");
		String authority=u.getAuthority();
		if (authority != null && authority.length() > 0) {
			result.append("//");
			int pos = authority.indexOf( '@' );
			if ( pos != -1 ){
				result.append(authority.substring(0,pos+1));
				authority = authority.substring(pos+1);
			}
			pos = authority.lastIndexOf(':');
			int	port = u.getPort();
			if ( port == -1 ){
				port = u.getDefaultPort();
			}
			if ( pos == -1 ){
				result.append(authority + ":" + port );
			}else{
				result.append(authority.substring(0,pos+1) + port );				
			}
		}

		try{
			return( new URL( result.toString()));
		}catch( Throwable e ){
			Debug.out(e);
			return(u);
		}
	}
	
	public static String
	getCanonicalString(
		URL		url )
	{
		String protocol = url.getProtocol();
		
		if ( !protocol.equals( protocol.toLowerCase( Locale.US ))){
			
			protocol = protocol.toLowerCase( Locale.US );
			
			url = UrlUtils.setProtocol( url, protocol ); 
		}
		
		int	port = url.getPort();
		
		if ( protocol.equals( "http" ) || protocol.equals( "https" )){
		
			if ( port == url.getDefaultPort()){
				
				url = UrlUtils.setPort( url, 0 );
			}
		}else{
			
			if ( port == -1 ){
				
				url = UrlUtils.setPort( url, url.getDefaultPort());
			}
		}
		
		return( url.toString());
	}
	
		/**
		 * Returns an explicit IPv4 url if the supplied one has both IPv6 and IPv4 addresses
		 * @param url
		 * @return
		 */
	
	public static URL
	getIPV4Fallback(
		URL	url )
	{
		try{
			InetAddress[] addresses = AddressUtils.getAllByName( url.getHost());
			
			if ( addresses.length > 0 ){
				
				InetAddress	ipv4	= null;
				InetAddress	ipv6	= null;
				
				for ( InetAddress a: addresses ){
					
					if ( a instanceof Inet4Address ){
						
						ipv4 = a;
						
					}else{
						
						ipv6 = a;
					}
				}
				
				if ( ipv4 != null && ipv6 != null ){
					
					url = UrlUtils.setHost( url, ipv4.getHostAddress());
					
					return( url );
				}
			}
		}catch( Throwable f ){
		}	
		
		return( null );
	}
	
	public static long
	getContentLength(
		URLConnection	con )
	{
		long res = con.getContentLength();
		
		if ( res == -1 ){
			
			try{
				String	str = con.getHeaderField( "content-length" );
				
				if ( str != null ){
					
					res = Long.parseLong( str );
				}
			}catch( Throwable e ){
				
			}
		}
		
		return( res );
	}
	
	
	
	public static void main(String[] args) {
		
		//MagnetURIHandler.getSingleton();
		
		System.out.println( URLEncoder.encode( "http://a.b.c/fred?a=10&b=20"));
		
		byte[] infohash = ByteFormatter.decodeString("1234567890123456789012345678901234567890");
		String[] test = {
				"http://moo.com",
				"http%3A%2F/moo%2Ecom",
				"magnet:?moo",
				"magnet%3A%3Fxt=urn:btih:26",
				"magnet%3A//%3Fmooo",
				"magnet:?xt=urn:btih:" + Base32.encode(infohash),
				"aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd",
				"magnet:?dn=OpenOffice.org_2.0.3_Win32Intel_install.exe&xt=urn:sha1:PEMIGLKMNFI4HZ4CCHZNPKZJNMAAORKN&xt=urn:tree:tiger:JMIJVWHCQUX47YYH7O4XIBCORNU2KYKHBBC6DHA&xt=urn:ed2k:1c0804541f34b6583a383bb8f2cec682&xl=96793015&xs=http://mirror.switch.ch/ftp/mirror/OpenOffice/stable/2.0.3/OOo_2.0.3_Win32Intel_install.exe%3Fa%3D10%26b%3D20",
				};
		for (int i = 0; i < test.length; i++) {
			System.out.println( test[i] );
			System.out.println("URLDecoder.decode: -> " + URLDecoder.decode(test[i]));
			System.out.println("decode:            -> " + decode(test[i]));
			System.out.println("decodeIf:          -> " + decodeIfNeeded(test[i]));
			System.out.println("isURL:             -> " + isURL(test[i]));
			System.out.println("parse:             -> " + parseTextForURL(test[i], true));
		}

		String[] testEncode = {
			"a b"
		};
		for (int i = 0; i < testEncode.length; i++) {
			String txt = testEncode[i];
			try {
				System.out.println("URLEncoder.encode: " + txt + " -> "
						+ URLEncoder.encode(txt, "UTF8"));
			} catch (UnsupportedEncodingException e) {
			}
			System.out.println("URLEncoder.encode: " + txt + " -> "
					+ URLEncoder.encode(txt));
			System.out.println("encode: " + txt + " -> " + encode(txt));
		}

	}
}
