sdklib/src/main/java/com/android/sdklib/internal/repository/UrlOpener.java - platform/tools/base - Git at Google

 /*
  * Copyright (C) 2011 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 package com.android.sdklib.internal.repository;

 import com.android.annotations.NonNull;
 import com.android.annotations.Nullable;
 import com.android.utils.Pair;

 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
 import org.apache.http.HttpStatus;
 import org.apache.http.ProtocolVersion;
 import org.apache.http.auth.AuthScope;
 import org.apache.http.auth.AuthState;
 import org.apache.http.auth.Credentials;
 import org.apache.http.auth.NTCredentials;
 import org.apache.http.auth.params.AuthPNames;
 import org.apache.http.client.ClientProtocolException;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.params.AuthPolicy;
 import org.apache.http.client.protocol.ClientContext;
 import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.http.impl.conn.ProxySelectorRoutePlanner;
 import org.apache.http.message.BasicHttpResponse;
 import org.apache.http.params.BasicHttpParams;
 import org.apache.http.params.HttpConnectionParams;
 import org.apache.http.params.HttpParams;
 import org.apache.http.protocol.BasicHttpContext;
 import org.apache.http.protocol.HttpContext;

 import java.io.BufferedInputStream;
 import java.io.ByteArrayInputStream;
 import java.io.FileNotFoundException;
 import java.io.FilterInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.HttpURLConnection;
 import java.net.Proxy;
 import java.net.ProxySelector;
 import java.net.URI;
 import java.net.URL;
 import java.net.URLConnection;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Properties;

 /**
  * This class holds static methods for downloading URL resources.
  * @see #openUrl(String, boolean, ITaskMonitor, Header[])
  * <p/>
  * Implementation detail: callers should use {@link DownloadCache} instead of this class.
  * {@link DownloadCache#openDirectUrl} is a direct pass-through to {@link UrlOpener} since
  * there's no caching. However from an implementation perspective it's still recommended
  * to pass down a {@link DownloadCache} instance, which will let us override the implementation
  * later on (for testing, for example.)
  *
  * @deprecated
  * com.android.sdklib.internal.repository has moved into Studio as
  * com.android.tools.idea.sdk.remote.internal.
  */
 @Deprecated
 class UrlOpener {

     private static final boolean DEBUG =
         System.getenv("ANDROID_DEBUG_URL_OPENER") != null; //$NON-NLS-1$

     private static Map<String, UserCredentials> sRealmCache =
             new HashMap<String, UserCredentials>();

     /** Timeout to establish a connection, in milliseconds. */
     private static int sConnectionTimeoutMs;
     /** Timeout waiting for data on a socket, in milliseconds. */
     private static int sSocketTimeoutMs;

     static {
         if (DEBUG) {
             Properties props = System.getProperties();
             for (String key : new String[] {
                     "http.proxyHost",           //$NON-NLS-1$
                     "http.proxyPort",           //$NON-NLS-1$
                     "https.proxyHost",          //$NON-NLS-1$
                     "https.proxyPort" }) {      //$NON-NLS-1$
                 String prop = props.getProperty(key);
                 if (prop != null) {
                     System.out.printf(
                             "SdkLib.UrlOpener Java.Prop %s='%s'\n",   //$NON-NLS-1$
                             key, prop);
                 }
             }
         }

         try {
             sConnectionTimeoutMs = Integer.parseInt(System.getenv("ANDROID_SDKMAN_CONN_TIMEOUT"));
         } catch (Exception ignore) {
             sConnectionTimeoutMs = 2 * 60 * 1000;
         }

         try {
             sSocketTimeoutMs = Integer.parseInt(System.getenv("ANDROID_SDKMAN_READ_TIMEOUT"));
         } catch (Exception ignore) {
             sSocketTimeoutMs = 1 * 60 * 1000;
         }
     }

     /**
      * This class cannot be instantiated.
      * @see #openUrl(String, boolean, ITaskMonitor, Header[])
      */
     private UrlOpener() {
     }

     /**
      * Opens a URL. It can be a simple URL or one which requires basic
      * authentication.
      * <p/>
      * Tries to access the given URL. If http response is either
      * {@code HttpStatus.SC_UNAUTHORIZED} or
      * {@code HttpStatus.SC_PROXY_AUTHENTICATION_REQUIRED}, asks for
      * login/password and tries to authenticate into proxy server and/or URL.
      * <p/>
      * This implementation relies on the Apache Http Client due to its
      * capabilities of proxy/http authentication. <br/>
      * Proxy configuration is determined by {@link ProxySelectorRoutePlanner} using the JVM proxy
      * settings by default.
      * <p/>
      * For more information see: <br/>
      * - {@code http://hc.apache.org/httpcomponents-client-ga/} <br/>
      * - {@code http://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/ProxySelectorRoutePlanner.html}
      * <p/>
      * There's a very simple realm cache implementation.
      * Login/Password for each realm are stored in a static {@link Map}.
      * Before asking the user the method verifies if the information is already
      * available in the memory cache.
      *
      * @param url the URL string to be opened.
      * @param needsMarkResetSupport Indicates the caller <em>must</em> have an input stream that
      *      supports the mark/reset operations (as indicated by {@link InputStream#markSupported()}.
      *      Implementation detail: If the original stream does not, it will be fetched and wrapped
      *      into a {@link ByteArrayInputStream}. This can only work sanely if the resource is a
      *      small file that can fit in memory. It also means the caller has no chance of showing
      *      a meaningful download progress. If unsure, callers should set this to false.
      * @param monitor {@link ITaskMonitor} to output status.
      * @param headers An optional array of HTTP headers to use in the GET request.
      * @return Returns a {@link Pair} with {@code first} holding an {@link InputStream}
      *      and {@code second} holding an {@link HttpResponse}.
      *      The returned pair is never null and contains
      *      at least a code; for http requests that provide them the response
      *      also contains locale, headers and an status line.
      *      The input stream can be null, especially in case of error.
      *      The caller must only accept the stream if the response code is 200 or similar.
      * @throws IOException Exception thrown when there are problems retrieving
      *             the URL or its content.
      * @throws CanceledByUserException Exception thrown if the user cancels the
      *              authentication dialog.
      */
     @NonNull
     static Pair<InputStream, HttpResponse> openUrl(
             @NonNull String url,
             boolean needsMarkResetSupport,
             @NonNull ITaskMonitor monitor,
             @Nullable Header[] headers)
         throws IOException, CanceledByUserException {

         Exception fallbackOnJavaUrlConnect = null;
         Pair<InputStream, HttpResponse> result = null;

         try {
             result = openWithHttpClient(url, monitor, headers);

         } catch (UnknownHostException e) {
             // Host in unknown. No need to even retry with the Url object,
             // if it's broken, it's broken. It's already an IOException but
             // it could use a better message.
             throw new IOException("Unknown Host " + e.getMessage(), e);

         } catch (ClientProtocolException e) {
             // We get this when HttpClient fails to accept the current protocol,
             // e.g. when processing file:// URLs.
             fallbackOnJavaUrlConnect = e;

         } catch (IOException e) {
             throw e;

         } catch (CanceledByUserException e) {
             // HTTP Basic Auth or NTLM login was canceled by user.
             throw e;

         } catch (Exception e) {
             if (DEBUG) {
                 System.out.printf("[HttpClient Error] %s : %s\n", url, e.toString());
             }

             fallbackOnJavaUrlConnect = e;
         }

         if (fallbackOnJavaUrlConnect != null) {
             // If the protocol is not supported by HttpClient (e.g. file:///),
             // revert to the standard java.net.Url.open.

             try {
                 result = openWithUrl(url, headers);
             } catch (IOException e) {
                 throw e;
             } catch (Exception e) {
                 if (DEBUG && !fallbackOnJavaUrlConnect.equals(e)) {
                     System.out.printf("[Url Error] %s : %s\n", url, e.toString());
                 }
             }
         }

         // If the caller requires an InputStream that supports mark/reset, let's
         // make sure we have such a stream.
         if (result != null && needsMarkResetSupport) {
             InputStream is = result.getFirst();
             if (is != null) {
                 if (!is.markSupported()) {
                     try {
                         // Consume the whole input stream and offer a byte array stream instead.
                         // This can only work sanely if the resource is a small file that can
                         // fit in memory. It also means the caller has no chance of showing
                         // a meaningful download progress.
                         InputStream is2 = toByteArrayInputStream(is);
                         if (is2 != null) {
                             result = Pair.of(is2, result.getSecond());
                             try {
                                 is.close();
                             } catch (Exception ignore) {}
                         }
                     } catch (Exception e3) {
                         // Ignore. If this can't work, caller will fail later.
                     }
                 }
             }
         }

         if (result == null) {
             // Make up an error code if we don't have one already.
             HttpResponse outResponse = new BasicHttpResponse(
                     new ProtocolVersion("HTTP", 1, 0),  //$NON-NLS-1$
                     HttpStatus.SC_METHOD_FAILURE, "");  //$NON-NLS-1$;  // 420=Method Failure
             result = Pair.of(null, outResponse);
         }

         return result;
     }

     // ByteArrayInputStream is the duct tape of input streams.
     private static InputStream toByteArrayInputStream(InputStream is) throws IOException {
         int inc = 4096;
         int curr = 0;
         byte[] result = new byte[inc];

         int n;
         while ((n = is.read(result, curr, result.length - curr)) != -1) {
             curr += n;
             if (curr == result.length) {
                 byte[] temp = new byte[curr + inc];
                 System.arraycopy(result, 0, temp, 0, curr);
                 result = temp;
             }
         }

         return new ByteArrayInputStream(result, 0, curr);
     }

     private static Pair<InputStream, HttpResponse> openWithUrl(
             String url,
             Header[] inHeaders) throws IOException {
         URL u = new URL(url);

         URLConnection c = u.openConnection();

         c.setConnectTimeout(sConnectionTimeoutMs);
         c.setReadTimeout(sSocketTimeoutMs);

         if (inHeaders != null) {
             for (Header header : inHeaders) {
                 c.setRequestProperty(header.getName(), header.getValue());
             }
         }

         // Trigger the access to the resource
         // (at which point setRequestProperty can't be used anymore.)
         int code = 200;

         if (c instanceof HttpURLConnection) {
             code = ((HttpURLConnection) c).getResponseCode();
         }

         // Get the input stream. That can fail for a file:// that doesn't exist
         // in which case we set the response code to 404.
         // Also we need a buffered input stream since the caller need to use is.reset().
         InputStream is = null;
         try {
             is = new BufferedInputStream(c.getInputStream());
         } catch (Exception ignore) {
             if (is == null && code == 200) {
                 code = 404;
             }
         }

         HttpResponse outResponse = new BasicHttpResponse(
                 new ProtocolVersion(u.getProtocol(),  1, 0), // make up the protocol version
                 code, "");  //$NON-NLS-1$;

         Map<String, List<String>> outHeaderMap = c.getHeaderFields();

         for (Entry<String, List<String>> entry : outHeaderMap.entrySet()) {
             String name = entry.getKey();
             if (name != null) {
                 List<String> values = entry.getValue();
                 if (!values.isEmpty()) {
                     outResponse.setHeader(name, values.get(0));
                 }
             }
         }

         return Pair.of(is, outResponse);
     }

     @NonNull
     private static Pair<InputStream, HttpResponse> openWithHttpClient(
             @NonNull String url,
             @NonNull ITaskMonitor monitor,
             Header[] inHeaders)
             throws IOException, ClientProtocolException, CanceledByUserException {
         UserCredentials result = null;
         String realm = null;

         HttpParams params = new BasicHttpParams();
         HttpConnectionParams.setConnectionTimeout(params, sConnectionTimeoutMs);
         HttpConnectionParams.setSoTimeout(params, sSocketTimeoutMs);

         // use the simple one
         final DefaultHttpClient httpClient = new DefaultHttpClient(params);


         // create local execution context
         HttpContext localContext = new BasicHttpContext();
         final HttpGet httpGet = new HttpGet(url);
         if (inHeaders != null) {
             for (Header header : inHeaders) {
                 httpGet.addHeader(header);
             }
         }

         // retrieve local java configured network in case there is the need to
         // authenticate a proxy
         ProxySelectorRoutePlanner routePlanner = new ProxySelectorRoutePlanner(
                     httpClient.getConnectionManager().getSchemeRegistry(),
                     ProxySelector.getDefault());
         httpClient.setRoutePlanner(routePlanner);

         // Set preference order for authentication options.
         // In particular, we don't add AuthPolicy.SPNEGO, which is given preference over NTLM in
         // servers that support both, as it is more secure. However, we don't seem to handle it
         // very well, so we leave it off the list.
         // See http://hc.apache.org/httpcomponents-client-ga/tutorial/html/authentication.html for
         // more info.
         List<String> authpref = new ArrayList<String>();
         authpref.add(AuthPolicy.BASIC);
         authpref.add(AuthPolicy.DIGEST);
         authpref.add(AuthPolicy.NTLM);
         httpClient.getParams().setParameter(AuthPNames.PROXY_AUTH_PREF, authpref);
         httpClient.getParams().setParameter(AuthPNames.TARGET_AUTH_PREF, authpref);

         if (DEBUG) {
             try {
                 URI uri = new URI(url);
                 ProxySelector sel = routePlanner.getProxySelector();
                 if (sel != null && uri.getScheme().startsWith("httP")) {               //$NON-NLS-1$
                     List<Proxy> list = sel.select(uri);
                     System.out.printf(
                             "SdkLib.UrlOpener:\n  Connect to: %s\n  Proxy List: %s\n", //$NON-NLS-1$
                             url,
                             list == null ? "(null)" : Arrays.toString(list.toArray()));//$NON-NLS-1$
                 }
             } catch (Exception e) {
                 System.out.printf(
                         "SdkLib.UrlOpener: Failed to get proxy info for %s: %s\n",     //$NON-NLS-1$
                         url, e.toString());
             }
         }

         boolean trying = true;
         // loop while the response is being fetched
         while (trying) {
             // connect and get status code
             HttpResponse response = httpClient.execute(httpGet, localContext);
             int statusCode = response.getStatusLine().getStatusCode();

             if (DEBUG) {
                 System.out.printf("  Status: %d\n", statusCode);                       //$NON-NLS-1$
             }

             // check whether any authentication is required
             AuthState authenticationState = null;
             if (statusCode == HttpStatus.SC_UNAUTHORIZED) {
                 // Target host authentication required
                 authenticationState = (AuthState) localContext
                         .getAttribute(ClientContext.TARGET_AUTH_STATE);
             }
             if (statusCode == HttpStatus.SC_PROXY_AUTHENTICATION_REQUIRED) {
                 // Proxy authentication required
                 authenticationState = (AuthState) localContext
                         .getAttribute(ClientContext.PROXY_AUTH_STATE);
             }
             if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_NOT_MODIFIED) {
                 // in case the status is OK and there is a realm and result,
                 // cache it
                 if (realm != null && result != null) {
                     sRealmCache.put(realm, result);
                 }
             }

             // there is the need for authentication
             if (authenticationState != null) {

                 // get scope and realm
                 AuthScope authScope = authenticationState.getAuthScope();

                 // If the current realm is different from the last one it means
                 // a pass was performed successfully to the last URL, therefore
                 // cache the last realm
                 if (realm != null && !realm.equals(authScope.getRealm())) {
                     sRealmCache.put(realm, result);
                 }

                 realm = authScope.getRealm();

                 // in case there is cache for this Realm, use it to authenticate
                 if (sRealmCache.containsKey(realm)) {
                     result = sRealmCache.get(realm);
                 } else {
                     // since there is no cache, request for login and password
                     result = monitor.displayLoginCredentialsPrompt("Site Authentication",
                             "Please login to the following domain: " + realm +
                             "\n\nServer requiring authentication:\n" + authScope.getHost());
                     if (result == null) {
                         throw new CanceledByUserException("User canceled login dialog.");
                     }
                 }

                 // retrieve authentication data
                 String user = result.getUserName();
                 String password = result.getPassword();
                 String workstation = result.getWorkstation();
                 String domain = result.getDomain();

                 // proceed in case there is indeed a user
                 if (user != null && !user.isEmpty()) {
                     Credentials credentials = new NTCredentials(user, password,
                             workstation, domain);
                     httpClient.getCredentialsProvider().setCredentials(authScope, credentials);
                     trying = true;
                 } else {
                     trying = false;
                 }
             } else {
                 trying = false;
             }

             HttpEntity entity = response.getEntity();

             if (entity != null) {
                 if (trying) {
                     // in case another pass to the Http Client will be performed, close the entity.
                     entity.getContent().close();
                 } else {
                     // since no pass to the Http Client is needed, retrieve the
                     // entity's content.

                     // Note: don't use something like a BufferedHttpEntity since it would consume
                     // all content and store it in memory, resulting in an OutOfMemory exception
                     // on a large download.
                     InputStream is = new FilterInputStream(entity.getContent()) {
                         @Override
                         public void close() throws IOException {
                             // Since Http Client is no longer needed, close it.

                             // Bug #21167: we need to tell http client to shutdown
                             // first, otherwise the super.close() would continue
                             // downloading and not return till complete.

                             httpClient.getConnectionManager().shutdown();
                             super.close();
                         }
                     };

                     HttpResponse outResponse = new BasicHttpResponse(response.getStatusLine());
                     outResponse.setHeaders(response.getAllHeaders());
                     outResponse.setLocale(response.getLocale());

                     return Pair.of(is, outResponse);
                 }
             } else if (statusCode == HttpStatus.SC_NOT_MODIFIED) {
                 // It's ok to not have an entity (e.g. nothing to download) for a 304
                 HttpResponse outResponse = new BasicHttpResponse(response.getStatusLine());
                 outResponse.setHeaders(response.getAllHeaders());
                 outResponse.setLocale(response.getLocale());

                 return Pair.of(null, outResponse);
             }
         }

         // We get here if we did not succeed. Callers do not expect a null result.
         httpClient.getConnectionManager().shutdown();
         throw new FileNotFoundException(url);
     }
 }
	/*
	* Copyright (C) 2011 The Android Open Source Project
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	package com.android.sdklib.internal.repository;

	import com.android.annotations.NonNull;
	import com.android.annotations.Nullable;
	import com.android.utils.Pair;

	import org.apache.http.Header;
	import org.apache.http.HttpEntity;
	import org.apache.http.HttpResponse;
	import org.apache.http.HttpStatus;
	import org.apache.http.ProtocolVersion;
	import org.apache.http.auth.AuthScope;
	import org.apache.http.auth.AuthState;
	import org.apache.http.auth.Credentials;
	import org.apache.http.auth.NTCredentials;
	import org.apache.http.auth.params.AuthPNames;
	import org.apache.http.client.ClientProtocolException;
	import org.apache.http.client.methods.HttpGet;
	import org.apache.http.client.params.AuthPolicy;
	import org.apache.http.client.protocol.ClientContext;
	import org.apache.http.impl.client.DefaultHttpClient;
	import org.apache.http.impl.conn.ProxySelectorRoutePlanner;
	import org.apache.http.message.BasicHttpResponse;
	import org.apache.http.params.BasicHttpParams;
	import org.apache.http.params.HttpConnectionParams;
	import org.apache.http.params.HttpParams;
	import org.apache.http.protocol.BasicHttpContext;
	import org.apache.http.protocol.HttpContext;

	import java.io.BufferedInputStream;
	import java.io.ByteArrayInputStream;
	import java.io.FileNotFoundException;
	import java.io.FilterInputStream;
	import java.io.IOException;
	import java.io.InputStream;
	import java.net.HttpURLConnection;
	import java.net.Proxy;
	import java.net.ProxySelector;
	import java.net.URI;
	import java.net.URL;
	import java.net.URLConnection;
	import java.net.UnknownHostException;
	import java.util.ArrayList;
	import java.util.Arrays;
	import java.util.HashMap;
	import java.util.List;
	import java.util.Map;
	import java.util.Map.Entry;
	import java.util.Properties;

	/**
	* This class holds static methods for downloading URL resources.
	* @see #openUrl(String, boolean, ITaskMonitor, Header[])
	* <p/>
	* Implementation detail: callers should use {@link DownloadCache} instead of this class.
	* {@link DownloadCache#openDirectUrl} is a direct pass-through to {@link UrlOpener} since
	* there's no caching. However from an implementation perspective it's still recommended
	* to pass down a {@link DownloadCache} instance, which will let us override the implementation
	* later on (for testing, for example.)
	*
	* @deprecated
	* com.android.sdklib.internal.repository has moved into Studio as
	* com.android.tools.idea.sdk.remote.internal.
	*/
	@Deprecated
	class UrlOpener {

	private static final boolean DEBUG =
	System.getenv("ANDROID_DEBUG_URL_OPENER") != null; //$NON-NLS-1$

	private static Map<String, UserCredentials> sRealmCache =
	new HashMap<String, UserCredentials>();

	/** Timeout to establish a connection, in milliseconds. */
	private static int sConnectionTimeoutMs;
	/** Timeout waiting for data on a socket, in milliseconds. */
	private static int sSocketTimeoutMs;

	static {
	if (DEBUG) {
	Properties props = System.getProperties();
	for (String key : new String[] {
	"http.proxyHost", //$NON-NLS-1$
	"http.proxyPort", //$NON-NLS-1$
	"https.proxyHost", //$NON-NLS-1$
	"https.proxyPort" }) { //$NON-NLS-1$
	String prop = props.getProperty(key);
	if (prop != null) {
	System.out.printf(
	"SdkLib.UrlOpener Java.Prop %s='%s'\n", //$NON-NLS-1$
	key, prop);
	}
	}
	}

	try {
	sConnectionTimeoutMs = Integer.parseInt(System.getenv("ANDROID_SDKMAN_CONN_TIMEOUT"));
	} catch (Exception ignore) {
	sConnectionTimeoutMs = 2 * 60 * 1000;
	}

	try {
	sSocketTimeoutMs = Integer.parseInt(System.getenv("ANDROID_SDKMAN_READ_TIMEOUT"));
	} catch (Exception ignore) {
	sSocketTimeoutMs = 1 * 60 * 1000;
	}
	}

	/**
	* This class cannot be instantiated.
	* @see #openUrl(String, boolean, ITaskMonitor, Header[])
	*/
	private UrlOpener() {
	}

	/**
	* Opens a URL. It can be a simple URL or one which requires basic
	* authentication.
	* <p/>
	* Tries to access the given URL. If http response is either
	* {@code HttpStatus.SC_UNAUTHORIZED} or
	* {@code HttpStatus.SC_PROXY_AUTHENTICATION_REQUIRED}, asks for
	* login/password and tries to authenticate into proxy server and/or URL.
	* <p/>
	* This implementation relies on the Apache Http Client due to its
	* capabilities of proxy/http authentication. <br/>
	* Proxy configuration is determined by {@link ProxySelectorRoutePlanner} using the JVM proxy
	* settings by default.
	* <p/>
	* For more information see: <br/>
	* - {@code http://hc.apache.org/httpcomponents-client-ga/} <br/>
	* - {@code http://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/ProxySelectorRoutePlanner.html}
	* <p/>
	* There's a very simple realm cache implementation.
	* Login/Password for each realm are stored in a static {@link Map}.
	* Before asking the user the method verifies if the information is already
	* available in the memory cache.
	*
	* @param url the URL string to be opened.
	* @param needsMarkResetSupport Indicates the caller <em>must</em> have an input stream that
	* supports the mark/reset operations (as indicated by {@link InputStream#markSupported()}.
	* Implementation detail: If the original stream does not, it will be fetched and wrapped
	* into a {@link ByteArrayInputStream}. This can only work sanely if the resource is a
	* small file that can fit in memory. It also means the caller has no chance of showing
	* a meaningful download progress. If unsure, callers should set this to false.
	* @param monitor {@link ITaskMonitor} to output status.
	* @param headers An optional array of HTTP headers to use in the GET request.
	* @return Returns a {@link Pair} with {@code first} holding an {@link InputStream}
	* and {@code second} holding an {@link HttpResponse}.
	* The returned pair is never null and contains
	* at least a code; for http requests that provide them the response
	* also contains locale, headers and an status line.
	* The input stream can be null, especially in case of error.
	* The caller must only accept the stream if the response code is 200 or similar.
	* @throws IOException Exception thrown when there are problems retrieving
	* the URL or its content.
	* @throws CanceledByUserException Exception thrown if the user cancels the
	* authentication dialog.
	*/
	@NonNull
	static Pair<InputStream, HttpResponse> openUrl(
	@NonNull String url,
	boolean needsMarkResetSupport,
	@NonNull ITaskMonitor monitor,
	@Nullable Header[] headers)
	throws IOException, CanceledByUserException {

	Exception fallbackOnJavaUrlConnect = null;
	Pair<InputStream, HttpResponse> result = null;

	try {
	result = openWithHttpClient(url, monitor, headers);

	} catch (UnknownHostException e) {
	// Host in unknown. No need to even retry with the Url object,
	// if it's broken, it's broken. It's already an IOException but
	// it could use a better message.
	throw new IOException("Unknown Host " + e.getMessage(), e);

	} catch (ClientProtocolException e) {
	// We get this when HttpClient fails to accept the current protocol,
	// e.g. when processing file:// URLs.
	fallbackOnJavaUrlConnect = e;

	} catch (IOException e) {
	throw e;

	} catch (CanceledByUserException e) {
	// HTTP Basic Auth or NTLM login was canceled by user.
	throw e;

	} catch (Exception e) {
	if (DEBUG) {
	System.out.printf("[HttpClient Error] %s : %s\n", url, e.toString());
	}

	fallbackOnJavaUrlConnect = e;
	}

	if (fallbackOnJavaUrlConnect != null) {
	// If the protocol is not supported by HttpClient (e.g. file:///),
	// revert to the standard java.net.Url.open.

	try {
	result = openWithUrl(url, headers);
	} catch (IOException e) {
	throw e;
	} catch (Exception e) {
	if (DEBUG && !fallbackOnJavaUrlConnect.equals(e)) {
	System.out.printf("[Url Error] %s : %s\n", url, e.toString());
	}
	}
	}

	// If the caller requires an InputStream that supports mark/reset, let's
	// make sure we have such a stream.
	if (result != null && needsMarkResetSupport) {
	InputStream is = result.getFirst();
	if (is != null) {
	if (!is.markSupported()) {
	try {
	// Consume the whole input stream and offer a byte array stream instead.
	// This can only work sanely if the resource is a small file that can
	// fit in memory. It also means the caller has no chance of showing
	// a meaningful download progress.
	InputStream is2 = toByteArrayInputStream(is);
	if (is2 != null) {
	result = Pair.of(is2, result.getSecond());
	try {
	is.close();
	} catch (Exception ignore) {}
	}
	} catch (Exception e3) {
	// Ignore. If this can't work, caller will fail later.
	}
	}
	}
	}

	if (result == null) {
	// Make up an error code if we don't have one already.
	HttpResponse outResponse = new BasicHttpResponse(
	new ProtocolVersion("HTTP", 1, 0), //$NON-NLS-1$
	HttpStatus.SC_METHOD_FAILURE, ""); //$NON-NLS-1$; // 420=Method Failure
	result = Pair.of(null, outResponse);
	}

	return result;
	}

	// ByteArrayInputStream is the duct tape of input streams.
	private static InputStream toByteArrayInputStream(InputStream is) throws IOException {
	int inc = 4096;
	int curr = 0;
	byte[] result = new byte[inc];

	int n;
	while ((n = is.read(result, curr, result.length - curr)) != -1) {
	curr += n;
	if (curr == result.length) {
	byte[] temp = new byte[curr + inc];
	System.arraycopy(result, 0, temp, 0, curr);
	result = temp;
	}
	}

	return new ByteArrayInputStream(result, 0, curr);
	}

	private static Pair<InputStream, HttpResponse> openWithUrl(
	String url,
	Header[] inHeaders) throws IOException {
	URL u = new URL(url);

	URLConnection c = u.openConnection();

	c.setConnectTimeout(sConnectionTimeoutMs);
	c.setReadTimeout(sSocketTimeoutMs);

	if (inHeaders != null) {
	for (Header header : inHeaders) {
	c.setRequestProperty(header.getName(), header.getValue());
	}
	}

	// Trigger the access to the resource
	// (at which point setRequestProperty can't be used anymore.)
	int code = 200;

	if (c instanceof HttpURLConnection) {
	code = ((HttpURLConnection) c).getResponseCode();
	}

	// Get the input stream. That can fail for a file:// that doesn't exist
	// in which case we set the response code to 404.
	// Also we need a buffered input stream since the caller need to use is.reset().
	InputStream is = null;
	try {
	is = new BufferedInputStream(c.getInputStream());
	} catch (Exception ignore) {
	if (is == null && code == 200) {
	code = 404;
	}
	}

	HttpResponse outResponse = new BasicHttpResponse(
	new ProtocolVersion(u.getProtocol(), 1, 0), // make up the protocol version
	code, ""); //$NON-NLS-1$;

	Map<String, List<String>> outHeaderMap = c.getHeaderFields();

	for (Entry<String, List<String>> entry : outHeaderMap.entrySet()) {
	String name = entry.getKey();
	if (name != null) {
	List<String> values = entry.getValue();
	if (!values.isEmpty()) {
	outResponse.setHeader(name, values.get(0));
	}
	}
	}

	return Pair.of(is, outResponse);
	}

	@NonNull
	private static Pair<InputStream, HttpResponse> openWithHttpClient(
	@NonNull String url,
	@NonNull ITaskMonitor monitor,
	Header[] inHeaders)
	throws IOException, ClientProtocolException, CanceledByUserException {
	UserCredentials result = null;
	String realm = null;

	HttpParams params = new BasicHttpParams();
	HttpConnectionParams.setConnectionTimeout(params, sConnectionTimeoutMs);
	HttpConnectionParams.setSoTimeout(params, sSocketTimeoutMs);

	// use the simple one
	final DefaultHttpClient httpClient = new DefaultHttpClient(params);


	// create local execution context
	HttpContext localContext = new BasicHttpContext();
	final HttpGet httpGet = new HttpGet(url);
	if (inHeaders != null) {
	for (Header header : inHeaders) {
	httpGet.addHeader(header);
	}
	}

	// retrieve local java configured network in case there is the need to
	// authenticate a proxy
	ProxySelectorRoutePlanner routePlanner = new ProxySelectorRoutePlanner(
	httpClient.getConnectionManager().getSchemeRegistry(),
	ProxySelector.getDefault());
	httpClient.setRoutePlanner(routePlanner);

	// Set preference order for authentication options.
	// In particular, we don't add AuthPolicy.SPNEGO, which is given preference over NTLM in
	// servers that support both, as it is more secure. However, we don't seem to handle it
	// very well, so we leave it off the list.
	// See http://hc.apache.org/httpcomponents-client-ga/tutorial/html/authentication.html for
	// more info.
	List<String> authpref = new ArrayList<String>();
	authpref.add(AuthPolicy.BASIC);
	authpref.add(AuthPolicy.DIGEST);
	authpref.add(AuthPolicy.NTLM);
	httpClient.getParams().setParameter(AuthPNames.PROXY_AUTH_PREF, authpref);
	httpClient.getParams().setParameter(AuthPNames.TARGET_AUTH_PREF, authpref);

	if (DEBUG) {
	try {
	URI uri = new URI(url);
	ProxySelector sel = routePlanner.getProxySelector();
	if (sel != null && uri.getScheme().startsWith("httP")) { //$NON-NLS-1$
	List<Proxy> list = sel.select(uri);
	System.out.printf(
	"SdkLib.UrlOpener:\n Connect to: %s\n Proxy List: %s\n", //$NON-NLS-1$
	url,
	list == null ? "(null)" : Arrays.toString(list.toArray()));//$NON-NLS-1$
	}
	} catch (Exception e) {
	System.out.printf(
	"SdkLib.UrlOpener: Failed to get proxy info for %s: %s\n", //$NON-NLS-1$
	url, e.toString());
	}
	}

	boolean trying = true;
	// loop while the response is being fetched
	while (trying) {
	// connect and get status code
	HttpResponse response = httpClient.execute(httpGet, localContext);
	int statusCode = response.getStatusLine().getStatusCode();

	if (DEBUG) {
	System.out.printf(" Status: %d\n", statusCode); //$NON-NLS-1$
	}

	// check whether any authentication is required
	AuthState authenticationState = null;
	if (statusCode == HttpStatus.SC_UNAUTHORIZED) {
	// Target host authentication required
	authenticationState = (AuthState) localContext
	.getAttribute(ClientContext.TARGET_AUTH_STATE);
	}
	if (statusCode == HttpStatus.SC_PROXY_AUTHENTICATION_REQUIRED) {
	// Proxy authentication required
	authenticationState = (AuthState) localContext
	.getAttribute(ClientContext.PROXY_AUTH_STATE);
	}
	if (statusCode == HttpStatus.SC_OK \|\| statusCode == HttpStatus.SC_NOT_MODIFIED) {
	// in case the status is OK and there is a realm and result,
	// cache it
	if (realm != null && result != null) {
	sRealmCache.put(realm, result);
	}
	}

	// there is the need for authentication
	if (authenticationState != null) {

	// get scope and realm
	AuthScope authScope = authenticationState.getAuthScope();

	// If the current realm is different from the last one it means
	// a pass was performed successfully to the last URL, therefore
	// cache the last realm
	if (realm != null && !realm.equals(authScope.getRealm())) {
	sRealmCache.put(realm, result);
	}

	realm = authScope.getRealm();

	// in case there is cache for this Realm, use it to authenticate
	if (sRealmCache.containsKey(realm)) {
	result = sRealmCache.get(realm);
	} else {
	// since there is no cache, request for login and password
	result = monitor.displayLoginCredentialsPrompt("Site Authentication",
	"Please login to the following domain: " + realm +
	"\n\nServer requiring authentication:\n" + authScope.getHost());
	if (result == null) {
	throw new CanceledByUserException("User canceled login dialog.");
	}
	}

	// retrieve authentication data
	String user = result.getUserName();
	String password = result.getPassword();
	String workstation = result.getWorkstation();
	String domain = result.getDomain();

	// proceed in case there is indeed a user
	if (user != null && !user.isEmpty()) {
	Credentials credentials = new NTCredentials(user, password,
	workstation, domain);
	httpClient.getCredentialsProvider().setCredentials(authScope, credentials);
	trying = true;
	} else {
	trying = false;
	}
	} else {
	trying = false;
	}

	HttpEntity entity = response.getEntity();

	if (entity != null) {
	if (trying) {
	// in case another pass to the Http Client will be performed, close the entity.
	entity.getContent().close();
	} else {
	// since no pass to the Http Client is needed, retrieve the
	// entity's content.

	// Note: don't use something like a BufferedHttpEntity since it would consume
	// all content and store it in memory, resulting in an OutOfMemory exception
	// on a large download.
	InputStream is = new FilterInputStream(entity.getContent()) {
	@Override
	public void close() throws IOException {
	// Since Http Client is no longer needed, close it.

	// Bug #21167: we need to tell http client to shutdown
	// first, otherwise the super.close() would continue
	// downloading and not return till complete.

	httpClient.getConnectionManager().shutdown();
	super.close();
	}
	};

	HttpResponse outResponse = new BasicHttpResponse(response.getStatusLine());
	outResponse.setHeaders(response.getAllHeaders());
	outResponse.setLocale(response.getLocale());

	return Pair.of(is, outResponse);
	}
	} else if (statusCode == HttpStatus.SC_NOT_MODIFIED) {
	// It's ok to not have an entity (e.g. nothing to download) for a 304
	HttpResponse outResponse = new BasicHttpResponse(response.getStatusLine());
	outResponse.setHeaders(response.getAllHeaders());
	outResponse.setLocale(response.getLocale());

	return Pair.of(null, outResponse);
	}
	}

	// We get here if we did not succeed. Callers do not expect a null result.
	httpClient.getConnectionManager().shutdown();
	throw new FileNotFoundException(url);
	}
	}