android/src/main/java/com/google/samples/apps/iosched/sync/RemoteConferenceDataFetcher.java - platform/external/iosched - Git at Google

 /*
  * Copyright 2014 Google Inc. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 package com.google.samples.apps.iosched.sync;

 import android.content.Context;
 import android.text.TextUtils;

 import com.google.samples.apps.iosched.Config;
 import com.google.gson.Gson;
 import com.google.samples.apps.iosched.R;
 import com.google.samples.apps.iosched.io.model.DataManifest;
 import com.google.samples.apps.iosched.util.FileUtils;
 import com.google.samples.apps.iosched.util.HashUtils;
 import com.google.samples.apps.iosched.util.TimeUtils;

 import java.io.File;
 import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.util.HashSet;
 import java.util.List;

 import com.turbomanage.httpclient.BasicHttpClient;
 import com.turbomanage.httpclient.ConsoleRequestLogger;
 import com.turbomanage.httpclient.HttpResponse;
 import com.turbomanage.httpclient.RequestLogger;

 import static com.google.samples.apps.iosched.util.LogUtils.*;

 /**
  * Helper class that fetches conference data from the remote server.
  */
 public class RemoteConferenceDataFetcher {
     private static final String TAG = makeLogTag(SyncHelper.class);

     // The directory under which we cache our downloaded files
     private static String CACHE_DIR = "data_cache";

     private Context mContext = null;

     // name of URL override file used for debug purposes
     private static final String URL_OVERRIDE_FILE_NAME = "iosched_manifest_url_override.txt";

     // URL of the remote manifest file
     private String mManifestUrl = null;

     // timestamp of the manifest file on the server
     private String mServerTimestamp = null;

     // the set of cache files we have used -- we use this for cache cleanup.
     private HashSet<String> mCacheFilesToKeep = new HashSet<String>();

     // total # of bytes downloaded (approximate)
     private long mBytesDownloaded = 0;

     // total # of bytes read from cache hits (approximate)
     private long mBytesReadFromCache = 0;

     public RemoteConferenceDataFetcher(Context context) {
         mContext = context;
         mManifestUrl = getManifestUrl();
     }

     /**
      * Fetches data from the remote server.
      *
      * @param refTimestamp The timestamp of the data to use as a reference; if the remote data
      *                     is not newer than this timestamp, no data will be downloaded and
      *                     this method will return null.
      *
      * @return The data downloaded, or null if there is no data to download
      * @throws IOException if an error occurred during download.
      */
     public String[] fetchConferenceDataIfNewer(String refTimestamp) throws IOException {
         if (TextUtils.isEmpty(mManifestUrl)) {
             LOGW(TAG, "Manifest URL is empty (remote sync disabled!).");
             return null;
         }

         BasicHttpClient httpClient = new BasicHttpClient();
         httpClient.setRequestLogger(mQuietLogger);

         // Only download if data is newer than refTimestamp
         // Cloud Storage is very picky with the If-Modified-Since format. If it's in a wrong
         // format, it refuses to serve the file, returning 400 HTTP error. So, if the
         // refTimestamp is in a wrong format, we simply ignore it. But pay attention to this
         // warning in the log, because it might mean unnecessary data is being downloaded.
         if (!TextUtils.isEmpty(refTimestamp)) {
             if (TimeUtils.isValidFormatForIfModifiedSinceHeader(refTimestamp)) {
                 httpClient.addHeader("If-Modified-Since", refTimestamp);
             } else {
                 LOGW(TAG, "Could not set If-Modified-Since HTTP header. Potentially downloading " +
                         "unnecessary data. Invalid format of refTimestamp argument: "+refTimestamp);
             }
         }

         HttpResponse response = httpClient.get(mManifestUrl, null);
         if (response == null) {
             LOGE(TAG, "Request for manifest returned null response.");
             throw new IOException("Request for data manifest returned null response.");
         }

         int status = response.getStatus();
         if (status == HttpURLConnection.HTTP_OK) {
             LOGD(TAG, "Server returned HTTP_OK, so new data is available.");
             mServerTimestamp = getLastModified(response);
             LOGD(TAG, "Server timestamp for new data is: " + mServerTimestamp);
             String body = response.getBodyAsString();
             if (TextUtils.isEmpty(body)) {
                 LOGE(TAG, "Request for manifest returned empty data.");
                 throw new IOException("Error fetching conference data manifest: no data.");
             }
             LOGD(TAG, "Manifest "+mManifestUrl+" read, contents: " + body);
             mBytesDownloaded += body.getBytes().length;
             return processManifest(body);
         } else if (status == HttpURLConnection.HTTP_NOT_MODIFIED) {
             // data on the server is not newer than our data
             LOGD(TAG, "HTTP_NOT_MODIFIED: data has not changed since " + refTimestamp);
             return null;
         } else {
             LOGE(TAG, "Error fetching conference data: HTTP status " + status);
             throw new IOException("Error fetching conference data: HTTP status " + status);
         }
     }

     // Returns the timestamp of the data downloaded from the server
     public String getServerDataTimestamp() {
         return mServerTimestamp;
     }

     /**
      * Returns the remote manifest file's URL. This is stored as a resource in the app,
      * but can be overriden by a file in the filesystem for debug purposes.
      * @return The URL of the remote manifest file.
      */
     private String getManifestUrl() {

         String manifestUrl = Config.MANIFEST_URL;

         // check for an override file
         File urlOverrideFile = new File(mContext.getFilesDir(), URL_OVERRIDE_FILE_NAME);
         if (urlOverrideFile.exists()) {
             try {
                 String overrideUrl = FileUtils.readFileAsString(urlOverrideFile).trim();
                 LOGW(TAG, "Debug URL override active: " + overrideUrl);
                 return overrideUrl;
             } catch (IOException ex) {
                 return manifestUrl;
             }
         } else {
             return manifestUrl;
         }
     }

     /**
      * Fetches a file from the cache/network, from an absolute or relative URL. If the
      * file is available in our cache, we read it from there; if not, we will
      * download it from the network and cache it.
      *
      * @param url The URL to fetch the file from. The URL may be absolute or relative; if
      *            relative, it will be considered to be relative to the manifest URL.
      * @return The contents of the file.
      * @throws IOException If an error occurs.
      */
     private String fetchFile(String url) throws IOException {
         // If this is a relative url, consider it relative to the manifest URL
         if (!url.contains("://")) {
             if (TextUtils.isEmpty(mManifestUrl) || !mManifestUrl.contains("/")) {
                 LOGE(TAG, "Could not build relative URL based on manifest URL.");
                 return null;
             }
             int i = mManifestUrl.lastIndexOf('/');
             url = mManifestUrl.substring(0, i) + "/" + url;
         }

         LOGD(TAG, "Attempting to fetch: " + sanitizeUrl(url));

         // Check if we have it in our cache first
         String body = null;
         try {
             body = loadFromCache(url);
             if (!TextUtils.isEmpty(body)) {
                 // cache hit
                 mBytesReadFromCache += body.getBytes().length;
                 mCacheFilesToKeep.add(getCacheKey(url));
                 return body;
             }
         } catch (IOException ex) {
             ex.printStackTrace();
             LOGE(TAG, "IOException getting file from cache.");
             // proceed anyway to attempt to download it from the network
         }

         // We don't have the file on cache, so download it
         LOGD(TAG, "Cache miss. Downloading from network: " + sanitizeUrl(url));
         BasicHttpClient client = new BasicHttpClient();
         client.setRequestLogger(mQuietLogger);
         HttpResponse response = client.get(url, null);

         if (response == null) {
             throw new IOException("Request for URL " + sanitizeUrl(url) + " returned null response.");
         }

         LOGD(TAG, "HTTP response " + response.getStatus());
         if (response.getStatus() == HttpURLConnection.HTTP_OK) {
             body = response.getBodyAsString();
             if (TextUtils.isEmpty(body)) {
                 throw new IOException("Got empty response when attempting to fetch " +
                         sanitizeUrl(url));
             }
             LOGD(TAG, "Successfully downloaded from network: " + sanitizeUrl(url));
             mBytesDownloaded += body.getBytes().length;
             writeToCache(url, body);
             mCacheFilesToKeep.add(getCacheKey(url));
             return body;
         } else {
             LOGE(TAG, "Failed to fetch from network: " + sanitizeUrl(url));
             throw new IOException("Request for URL " + sanitizeUrl(url) +
                     " failed with HTTP error " + response.getStatus());
         }
     }

     /**
      * Returns the cache file where we store our cache of the response of the given URL.
      * @param url The URL for which to return the cache file.
      * @return The cache file.
      */
     private File getCacheFile(String url) {
         String cacheKey = getCacheKey(url);
         return new File(mContext.getCacheDir() + File.separator + CACHE_DIR + File.separator +
                 cacheKey);
     }

     // Creates the cache directory, if it doesn't exist yet
     private void createCacheDir() throws IOException {
         File dir = new File(mContext.getCacheDir() + File.separator + CACHE_DIR);
         if (!dir.exists() && !dir.mkdir()) {
             throw new IOException("Failed to mkdir: " + dir);
         }
     }


     /**
      * Loads our cached content corresponding to the given URL.
      * @param url The URL for which to load the cached response.
      * @return The cached response corresponding to the URL; or null if the given URL
      * does not exist in our cache.
      * @throws IOException If there is an error reading the cache.
      */
     private String loadFromCache(String url) throws IOException {
         String cacheKey = getCacheKey(url);
         File cacheFile = getCacheFile(url);
         if (cacheFile.exists()) {
             LOGD(TAG, "Cache hit " + cacheKey + " for " + sanitizeUrl(url));
             return FileUtils.readFileAsString(cacheFile);
         } else {
             LOGD(TAG, "Cache miss " + cacheKey + " for " + sanitizeUrl(url));
             return null;
         }
     }

     /**
      * Writes a file to the cache.
      * @param url The URL from which the contents were retrieved.
      * @param body The contents retrieved from the given URL.
      * @throws IOException If there is a problem writing the file.
      */
     private void writeToCache(String url, String body) throws IOException {
         String cacheKey = getCacheKey(url);
         File cacheFile = getCacheFile(url);
         createCacheDir();
         FileUtils.writeFile(body, cacheFile);
         LOGD(TAG, "Wrote to cache " + cacheKey + " --> " + sanitizeUrl(url));
     }

     /**
      * Returns the cache key to be used to store the given URL. The cache key is the
      * file name under which the contents of the URL are stored.
      * @param url The URL.
      * @return The cache key (guaranteed to be a valid filename)
      */
     private String getCacheKey(String url) {
         return HashUtils.computeWeakHash(url.trim()) + String.format("%04x", url.length());
     }

     // Sanitize a URL for logging purposes (only the last component is left visible).
     private String sanitizeUrl(String url) {
         int i = url.lastIndexOf('/');
         if (i >= 0 && i < url.length()) {
             return url.substring(0, i).replaceAll("[A-za-z]", "*") +
                     url.substring(i);
         }
         else return url.replaceAll("[A-za-z]", "*");
     }

     private static final String MANIFEST_FORMAT = "iosched-json-v1";

     /**
      * Process the data manifest and download data files referenced from it.
      * @param manifestJson The JSON of the manifest file.
      * @return The contents of the set of files referenced from the manifest, or null
      * if none could be retrieved.
      * @throws IOException If an error occurs while retrieving information.
      */
     private String[] processManifest(String manifestJson) throws IOException {
         LOGD(TAG, "Processing data manifest, length " + manifestJson.length());

         DataManifest manifest = new Gson().fromJson(manifestJson, DataManifest.class);
         if (manifest.format == null || !manifest.format.equals(MANIFEST_FORMAT)) {
             LOGE(TAG, "Manifest has invalid format spec: " + manifest.format);
             throw new IOException("Invalid format spec on manifest:" + manifest.format);
         }

         if (manifest.data_files == null || manifest.data_files.length == 0) {
             LOGW(TAG, "Manifest does not list any files. Nothing done.");
             return null;
         }

         LOGD(TAG, "Manifest lists " + manifest.data_files.length + " data files.");
         String[] jsons = new String[manifest.data_files.length];
         for (int i = 0; i < manifest.data_files.length; i++) {
             String url = manifest.data_files[i];
             LOGD(TAG, "Processing data file: " + sanitizeUrl(url));
             jsons[i] = fetchFile(url);
             if (TextUtils.isEmpty(jsons[i])) {
                 LOGE(TAG, "Failed to fetch data file: " + sanitizeUrl(url));
                 throw new IOException("Failed to fetch data file " + sanitizeUrl(url));
             }
         }

         LOGD(TAG, "Got " + jsons.length + " data files.");
         cleanUpCache();
         return jsons;
     }

     // Delete unnecessary files from our cache
     private void cleanUpCache() {
         LOGD(TAG, "Starting cache cleanup, " + mCacheFilesToKeep.size() + " URLs to keep.");
         File dir = new File(mContext.getCacheDir() + File.separator + CACHE_DIR);
         if (!dir.exists()) {
             LOGD(TAG, "Cleanup complete (there is no cache).");
             return;
         }

         int deleted = 0, kept = 0;
         for (File file : dir.listFiles()) {
             if (mCacheFilesToKeep.contains(file.getName())) {
                 LOGD(TAG, "Cache cleanup: KEEEPING " + file.getName());
                 ++kept;
             } else {
                 LOGD(TAG, "Cache cleanup: DELETING " + file.getName());
                 file.delete();
                 ++deleted;
             }
         }

         LOGD(TAG, "End of cache cleanup. " + kept + " files kept, " + deleted + " deleted.");
     }

     public long getTotalBytesDownloaded() {
         return mBytesDownloaded;
     }

     public long getTotalBytesReadFromCache() {
         return mBytesReadFromCache;
     }

     private String getLastModified(HttpResponse resp) {
         if (!resp.getHeaders().containsKey("Last-Modified")) {
             return "";
         }

         List<String> s = resp.getHeaders().get("Last-Modified");
         return s.isEmpty() ? "" : s.get(0);
     }

     /**
      * A type of ConsoleRequestLogger that does not log requests and responses.
      */
     private RequestLogger mQuietLogger = new ConsoleRequestLogger(){
         @Override
         public void logRequest(HttpURLConnection uc, Object content) throws IOException { }

         @Override
         public void logResponse(HttpResponse res) { }
     };


 }
	/*
	* Copyright 2014 Google Inc. All rights reserved.
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	package com.google.samples.apps.iosched.sync;

	import android.content.Context;
	import android.text.TextUtils;

	import com.google.samples.apps.iosched.Config;
	import com.google.gson.Gson;
	import com.google.samples.apps.iosched.R;
	import com.google.samples.apps.iosched.io.model.DataManifest;
	import com.google.samples.apps.iosched.util.FileUtils;
	import com.google.samples.apps.iosched.util.HashUtils;
	import com.google.samples.apps.iosched.util.TimeUtils;

	import java.io.File;
	import java.io.IOException;
	import java.net.HttpURLConnection;
	import java.util.HashSet;
	import java.util.List;

	import com.turbomanage.httpclient.BasicHttpClient;
	import com.turbomanage.httpclient.ConsoleRequestLogger;
	import com.turbomanage.httpclient.HttpResponse;
	import com.turbomanage.httpclient.RequestLogger;

	import static com.google.samples.apps.iosched.util.LogUtils.*;

	/**
	* Helper class that fetches conference data from the remote server.
	*/
	public class RemoteConferenceDataFetcher {
	private static final String TAG = makeLogTag(SyncHelper.class);

	// The directory under which we cache our downloaded files
	private static String CACHE_DIR = "data_cache";

	private Context mContext = null;

	// name of URL override file used for debug purposes
	private static final String URL_OVERRIDE_FILE_NAME = "iosched_manifest_url_override.txt";

	// URL of the remote manifest file
	private String mManifestUrl = null;

	// timestamp of the manifest file on the server
	private String mServerTimestamp = null;

	// the set of cache files we have used -- we use this for cache cleanup.
	private HashSet<String> mCacheFilesToKeep = new HashSet<String>();

	// total # of bytes downloaded (approximate)
	private long mBytesDownloaded = 0;

	// total # of bytes read from cache hits (approximate)
	private long mBytesReadFromCache = 0;

	public RemoteConferenceDataFetcher(Context context) {
	mContext = context;
	mManifestUrl = getManifestUrl();
	}

	/**
	* Fetches data from the remote server.
	*
	* @param refTimestamp The timestamp of the data to use as a reference; if the remote data
	* is not newer than this timestamp, no data will be downloaded and
	* this method will return null.
	*
	* @return The data downloaded, or null if there is no data to download
	* @throws IOException if an error occurred during download.
	*/
	public String[] fetchConferenceDataIfNewer(String refTimestamp) throws IOException {
	if (TextUtils.isEmpty(mManifestUrl)) {
	LOGW(TAG, "Manifest URL is empty (remote sync disabled!).");
	return null;
	}

	BasicHttpClient httpClient = new BasicHttpClient();
	httpClient.setRequestLogger(mQuietLogger);

	// Only download if data is newer than refTimestamp
	// Cloud Storage is very picky with the If-Modified-Since format. If it's in a wrong
	// format, it refuses to serve the file, returning 400 HTTP error. So, if the
	// refTimestamp is in a wrong format, we simply ignore it. But pay attention to this
	// warning in the log, because it might mean unnecessary data is being downloaded.
	if (!TextUtils.isEmpty(refTimestamp)) {
	if (TimeUtils.isValidFormatForIfModifiedSinceHeader(refTimestamp)) {
	httpClient.addHeader("If-Modified-Since", refTimestamp);
	} else {
	LOGW(TAG, "Could not set If-Modified-Since HTTP header. Potentially downloading " +
	"unnecessary data. Invalid format of refTimestamp argument: "+refTimestamp);
	}
	}

	HttpResponse response = httpClient.get(mManifestUrl, null);
	if (response == null) {
	LOGE(TAG, "Request for manifest returned null response.");
	throw new IOException("Request for data manifest returned null response.");
	}

	int status = response.getStatus();
	if (status == HttpURLConnection.HTTP_OK) {
	LOGD(TAG, "Server returned HTTP_OK, so new data is available.");
	mServerTimestamp = getLastModified(response);
	LOGD(TAG, "Server timestamp for new data is: " + mServerTimestamp);
	String body = response.getBodyAsString();
	if (TextUtils.isEmpty(body)) {
	LOGE(TAG, "Request for manifest returned empty data.");
	throw new IOException("Error fetching conference data manifest: no data.");
	}
	LOGD(TAG, "Manifest "+mManifestUrl+" read, contents: " + body);
	mBytesDownloaded += body.getBytes().length;
	return processManifest(body);
	} else if (status == HttpURLConnection.HTTP_NOT_MODIFIED) {
	// data on the server is not newer than our data
	LOGD(TAG, "HTTP_NOT_MODIFIED: data has not changed since " + refTimestamp);
	return null;
	} else {
	LOGE(TAG, "Error fetching conference data: HTTP status " + status);
	throw new IOException("Error fetching conference data: HTTP status " + status);
	}
	}

	// Returns the timestamp of the data downloaded from the server
	public String getServerDataTimestamp() {
	return mServerTimestamp;
	}

	/**
	* Returns the remote manifest file's URL. This is stored as a resource in the app,
	* but can be overriden by a file in the filesystem for debug purposes.
	* @return The URL of the remote manifest file.
	*/
	private String getManifestUrl() {

	String manifestUrl = Config.MANIFEST_URL;

	// check for an override file
	File urlOverrideFile = new File(mContext.getFilesDir(), URL_OVERRIDE_FILE_NAME);
	if (urlOverrideFile.exists()) {
	try {
	String overrideUrl = FileUtils.readFileAsString(urlOverrideFile).trim();
	LOGW(TAG, "Debug URL override active: " + overrideUrl);
	return overrideUrl;
	} catch (IOException ex) {
	return manifestUrl;
	}
	} else {
	return manifestUrl;
	}
	}

	/**
	* Fetches a file from the cache/network, from an absolute or relative URL. If the
	* file is available in our cache, we read it from there; if not, we will
	* download it from the network and cache it.
	*
	* @param url The URL to fetch the file from. The URL may be absolute or relative; if
	* relative, it will be considered to be relative to the manifest URL.
	* @return The contents of the file.
	* @throws IOException If an error occurs.
	*/
	private String fetchFile(String url) throws IOException {
	// If this is a relative url, consider it relative to the manifest URL
	if (!url.contains("://")) {
	if (TextUtils.isEmpty(mManifestUrl) \|\| !mManifestUrl.contains("/")) {
	LOGE(TAG, "Could not build relative URL based on manifest URL.");
	return null;
	}
	int i = mManifestUrl.lastIndexOf('/');
	url = mManifestUrl.substring(0, i) + "/" + url;
	}

	LOGD(TAG, "Attempting to fetch: " + sanitizeUrl(url));

	// Check if we have it in our cache first
	String body = null;
	try {
	body = loadFromCache(url);
	if (!TextUtils.isEmpty(body)) {
	// cache hit
	mBytesReadFromCache += body.getBytes().length;
	mCacheFilesToKeep.add(getCacheKey(url));
	return body;
	}
	} catch (IOException ex) {
	ex.printStackTrace();
	LOGE(TAG, "IOException getting file from cache.");
	// proceed anyway to attempt to download it from the network
	}

	// We don't have the file on cache, so download it
	LOGD(TAG, "Cache miss. Downloading from network: " + sanitizeUrl(url));
	BasicHttpClient client = new BasicHttpClient();
	client.setRequestLogger(mQuietLogger);
	HttpResponse response = client.get(url, null);

	if (response == null) {
	throw new IOException("Request for URL " + sanitizeUrl(url) + " returned null response.");
	}

	LOGD(TAG, "HTTP response " + response.getStatus());
	if (response.getStatus() == HttpURLConnection.HTTP_OK) {
	body = response.getBodyAsString();
	if (TextUtils.isEmpty(body)) {
	throw new IOException("Got empty response when attempting to fetch " +
	sanitizeUrl(url));
	}
	LOGD(TAG, "Successfully downloaded from network: " + sanitizeUrl(url));
	mBytesDownloaded += body.getBytes().length;
	writeToCache(url, body);
	mCacheFilesToKeep.add(getCacheKey(url));
	return body;
	} else {
	LOGE(TAG, "Failed to fetch from network: " + sanitizeUrl(url));
	throw new IOException("Request for URL " + sanitizeUrl(url) +
	" failed with HTTP error " + response.getStatus());
	}
	}

	/**
	* Returns the cache file where we store our cache of the response of the given URL.
	* @param url The URL for which to return the cache file.
	* @return The cache file.
	*/
	private File getCacheFile(String url) {
	String cacheKey = getCacheKey(url);
	return new File(mContext.getCacheDir() + File.separator + CACHE_DIR + File.separator +
	cacheKey);
	}

	// Creates the cache directory, if it doesn't exist yet
	private void createCacheDir() throws IOException {
	File dir = new File(mContext.getCacheDir() + File.separator + CACHE_DIR);
	if (!dir.exists() && !dir.mkdir()) {
	throw new IOException("Failed to mkdir: " + dir);
	}
	}


	/**
	* Loads our cached content corresponding to the given URL.
	* @param url The URL for which to load the cached response.
	* @return The cached response corresponding to the URL; or null if the given URL
	* does not exist in our cache.
	* @throws IOException If there is an error reading the cache.
	*/
	private String loadFromCache(String url) throws IOException {
	String cacheKey = getCacheKey(url);
	File cacheFile = getCacheFile(url);
	if (cacheFile.exists()) {
	LOGD(TAG, "Cache hit " + cacheKey + " for " + sanitizeUrl(url));
	return FileUtils.readFileAsString(cacheFile);
	} else {
	LOGD(TAG, "Cache miss " + cacheKey + " for " + sanitizeUrl(url));
	return null;
	}
	}

	/**
	* Writes a file to the cache.
	* @param url The URL from which the contents were retrieved.
	* @param body The contents retrieved from the given URL.
	* @throws IOException If there is a problem writing the file.
	*/
	private void writeToCache(String url, String body) throws IOException {
	String cacheKey = getCacheKey(url);
	File cacheFile = getCacheFile(url);
	createCacheDir();
	FileUtils.writeFile(body, cacheFile);
	LOGD(TAG, "Wrote to cache " + cacheKey + " --> " + sanitizeUrl(url));
	}

	/**
	* Returns the cache key to be used to store the given URL. The cache key is the
	* file name under which the contents of the URL are stored.
	* @param url The URL.
	* @return The cache key (guaranteed to be a valid filename)
	*/
	private String getCacheKey(String url) {
	return HashUtils.computeWeakHash(url.trim()) + String.format("%04x", url.length());
	}

	// Sanitize a URL for logging purposes (only the last component is left visible).
	private String sanitizeUrl(String url) {
	int i = url.lastIndexOf('/');
	if (i >= 0 && i < url.length()) {
	return url.substring(0, i).replaceAll("[A-za-z]", "*") +
	url.substring(i);
	}
	else return url.replaceAll("[A-za-z]", "*");
	}

	private static final String MANIFEST_FORMAT = "iosched-json-v1";

	/**
	* Process the data manifest and download data files referenced from it.
	* @param manifestJson The JSON of the manifest file.
	* @return The contents of the set of files referenced from the manifest, or null
	* if none could be retrieved.
	* @throws IOException If an error occurs while retrieving information.
	*/
	private String[] processManifest(String manifestJson) throws IOException {
	LOGD(TAG, "Processing data manifest, length " + manifestJson.length());

	DataManifest manifest = new Gson().fromJson(manifestJson, DataManifest.class);
	if (manifest.format == null \|\| !manifest.format.equals(MANIFEST_FORMAT)) {
	LOGE(TAG, "Manifest has invalid format spec: " + manifest.format);
	throw new IOException("Invalid format spec on manifest:" + manifest.format);
	}

	if (manifest.data_files == null \|\| manifest.data_files.length == 0) {
	LOGW(TAG, "Manifest does not list any files. Nothing done.");
	return null;
	}

	LOGD(TAG, "Manifest lists " + manifest.data_files.length + " data files.");
	String[] jsons = new String[manifest.data_files.length];
	for (int i = 0; i < manifest.data_files.length; i++) {
	String url = manifest.data_files[i];
	LOGD(TAG, "Processing data file: " + sanitizeUrl(url));
	jsons[i] = fetchFile(url);
	if (TextUtils.isEmpty(jsons[i])) {
	LOGE(TAG, "Failed to fetch data file: " + sanitizeUrl(url));
	throw new IOException("Failed to fetch data file " + sanitizeUrl(url));
	}
	}

	LOGD(TAG, "Got " + jsons.length + " data files.");
	cleanUpCache();
	return jsons;
	}

	// Delete unnecessary files from our cache
	private void cleanUpCache() {
	LOGD(TAG, "Starting cache cleanup, " + mCacheFilesToKeep.size() + " URLs to keep.");
	File dir = new File(mContext.getCacheDir() + File.separator + CACHE_DIR);
	if (!dir.exists()) {
	LOGD(TAG, "Cleanup complete (there is no cache).");
	return;
	}

	int deleted = 0, kept = 0;
	for (File file : dir.listFiles()) {
	if (mCacheFilesToKeep.contains(file.getName())) {
	LOGD(TAG, "Cache cleanup: KEEEPING " + file.getName());
	++kept;
	} else {
	LOGD(TAG, "Cache cleanup: DELETING " + file.getName());
	file.delete();
	++deleted;
	}
	}

	LOGD(TAG, "End of cache cleanup. " + kept + " files kept, " + deleted + " deleted.");
	}

	public long getTotalBytesDownloaded() {
	return mBytesDownloaded;
	}

	public long getTotalBytesReadFromCache() {
	return mBytesReadFromCache;
	}

	private String getLastModified(HttpResponse resp) {
	if (!resp.getHeaders().containsKey("Last-Modified")) {
	return "";
	}

	List<String> s = resp.getHeaders().get("Last-Modified");
	return s.isEmpty() ? "" : s.get(0);
	}

	/**
	* A type of ConsoleRequestLogger that does not log requests and responses.
	*/
	private RequestLogger mQuietLogger = new ConsoleRequestLogger(){
	@Override
	public void logRequest(HttpURLConnection uc, Object content) throws IOException { }

	@Override
	public void logResponse(HttpResponse res) { }
	};


	}