Split Apply Ops Into Chuncks
To prevent TransactionTooLargeException errors. If it still fails, retry with
a minimal chunk to apply one message at a time.
Bug: 10402517
Change-Id: Ia121652da6da56587939dc813abb801d055c24ca
diff --git a/src/com/android/exchange/Eas.java b/src/com/android/exchange/Eas.java
index 7f9997f..396ec29 100644
--- a/src/com/android/exchange/Eas.java
+++ b/src/com/android/exchange/Eas.java
@@ -88,6 +88,8 @@
public static final String MIME_BODY_PREFERENCE_TEXT = "0";
public static final String MIME_BODY_PREFERENCE_MIME = "2";
+ // These limits must never exceed about 500k which is half the max size of a Binder IPC buffer.
+
// For EAS 12, we use HTML, so we want a larger size than in EAS 2.5
public static final String EAS12_TRUNCATION_SIZE = "200000";
// For EAS 2.5, truncation is a code; the largest is "7", which is 100k
diff --git a/src/com/android/exchange/adapter/AbstractSyncParser.java b/src/com/android/exchange/adapter/AbstractSyncParser.java
index 118c704..a3a0633 100644
--- a/src/com/android/exchange/adapter/AbstractSyncParser.java
+++ b/src/com/android/exchange/adapter/AbstractSyncParser.java
@@ -20,12 +20,15 @@
import android.content.ContentResolver;
import android.content.ContentValues;
import android.content.Context;
+import android.content.OperationApplicationException;
+import android.os.RemoteException;
import com.android.emailcommon.provider.Account;
import com.android.emailcommon.provider.EmailContent.MailboxColumns;
import com.android.emailcommon.provider.Mailbox;
import com.android.exchange.CommandStatusException;
import com.android.exchange.CommandStatusException.CommandStatus;
+import com.android.mail.utils.LogUtils;
import java.io.IOException;
import java.io.InputStream;
@@ -37,6 +40,8 @@
*
*/
public abstract class AbstractSyncParser extends Parser {
+ private static final String TAG = "AbstractSyncParser";
+
protected Mailbox mMailbox;
protected Account mAccount;
protected Context mContext;
@@ -90,7 +95,8 @@
* Commit any changes found during parsing
* @throws IOException
*/
- public abstract void commit() throws IOException;
+ public abstract void commit() throws IOException, RemoteException,
+ OperationApplicationException;
public boolean isLooping() {
return mLooping;
@@ -194,30 +200,16 @@
}
// Commit any changes
- commit();
-
-
- // TODO: I don't think this is still relevant. Syncing should not trigger changes in the
- // sync interval.
- /*
- // If the sync interval has changed, we need to save it
- if (mMailbox.mSyncInterval != interval) {
- cv.put(MailboxColumns.SYNC_INTERVAL, mMailbox.mSyncInterval);
- mailboxUpdated = true;
- // If there are changes, and we were bounced from push/ping, try again
- } else if (mAdapter.mChangeCount > 0 &&
- mAccount.mSyncInterval == Account.CHECK_INTERVAL_PUSH &&
- mMailbox.mSyncInterval > 0) {
- userLog("Changes found to ping loop mailbox ", mMailbox.mDisplayName, ": will ping.");
- cv.put(MailboxColumns.SYNC_INTERVAL, Mailbox.CHECK_INTERVAL_PING);
- mailboxUpdated = true;
- abortSyncs = true;
+ try {
+ commit();
+ if (mailboxUpdated) {
+ mMailbox.update(mContext, cv);
+ }
+ } catch (RemoteException e) {
+ LogUtils.e(TAG, "Failed to commit changes", e);
+ } catch (OperationApplicationException e) {
+ LogUtils.e(TAG, "Failed to commit changes", e);
}
- */
- if (mailboxUpdated) {
- mMailbox.update(mContext, cv);
- }
-
// Let the caller know that there's more to do
if (moreAvailable) {
userLog("MoreAvailable");
diff --git a/src/com/android/exchange/adapter/EmailSyncParser.java b/src/com/android/exchange/adapter/EmailSyncParser.java
index 8024b98..76dd163 100644
--- a/src/com/android/exchange/adapter/EmailSyncParser.java
+++ b/src/com/android/exchange/adapter/EmailSyncParser.java
@@ -7,6 +7,7 @@
import android.content.Context;
import android.content.OperationApplicationException;
import android.database.Cursor;
+import android.os.Parcel;
import android.os.RemoteException;
import android.os.TransactionTooLargeException;
import android.provider.CalendarContract;
@@ -14,6 +15,7 @@
import android.text.SpannedString;
import android.text.TextUtils;
import android.util.Base64;
+import android.util.Log;
import android.webkit.MimeTypeMap;
import com.android.emailcommon.internet.MimeMessage;
@@ -76,6 +78,14 @@
static final int LAST_VERB_FORWARD = 3;
private final Policy mPolicy;
+
+ // Max times to retry when we get a TransactionTooLargeException exception
+ private static final int MAX_RETRIES = 10;
+
+ // Max number of ops per batch. It could end up more than this but once we detect we are at or
+ // above this number, we flush.
+ private static final int MAX_OPS_PER_BATCH = 50;
+
private boolean mFetchNeeded = false;
private final Map<String, Integer> mMessageUpdateStatus = new HashMap();
@@ -719,25 +729,45 @@
}
}
+ /**
+ * Commit all changes. This results in a Binder IPC call which has constraint on the size of
+ * the data, the docs say it currently 1MB. We set a limit to the size of the message we fetch
+ * with {@link Eas#EAS12_TRUNCATION_SIZE} & {@link Eas#EAS12_TRUNCATION_SIZE} which are at 200k
+ * or bellow. As long as these limits are bellow 500k, we should be able to apply a single
+ * message (the transaction size is about double the message size because Java strings are 16
+ * bit.
+ * <b/>
+ * We first try to apply the changes in normal chunk size {@link #MAX_OPS_PER_BATCH}. If we get
+ * a {@link TransactionTooLargeException} we try again with but this time, we apply each change
+ * immediately.
+ */
@Override
- public void commit() {
- commitImpl(0);
+ public void commit() throws RemoteException, OperationApplicationException {
+ try {
+ commitImpl(MAX_OPS_PER_BATCH);
+ } catch (TransactionTooLargeException e) {
+ // Try again but apply batch after every message. The max message size defined in
+ // Eas.EAS12_TRUNCATION_SIZE or Eas.EAS2_5_TRUNCATION_SIZE is small enough to fit
+ // in a single Binder call.
+ LogUtils.w(TAG, "Transaction too large, retrying in single mode", e);
+ commitImpl(1);
+ }
}
- public void commitImpl(int tryCount) {
+ public void commitImpl(int maxOpsPerBatch)
+ throws RemoteException, OperationApplicationException {
// Use a batch operation to handle the changes
ArrayList<ContentProviderOperation> ops = new ArrayList<ContentProviderOperation>();
// Maximum size of message text per fetch
int numFetched = fetchedEmails.size();
- int maxPerFetch = 0;
- if (numFetched > 0 && tryCount > 0) {
- // Educated guess that 450000 chars (900k) is ok; 600k is a killer
- // Remember that when fetching, we're not getting any other data
- // We'll keep trying, reducing the maximum each time
- // Realistically, this will rarely exceed 1, and probably never 2
- maxPerFetch = 450000 / numFetched / tryCount;
- }
+ LogUtils.d(TAG, "commitImpl: maxOpsPerBatch=%d numFetched=%d numNew=%d "
+ + "numDeleted=%d numChanged=%d",
+ maxOpsPerBatch,
+ numFetched,
+ newEmails.size(),
+ deletedEmails.size(),
+ changedEmails.size());
for (EmailContent.Message msg: fetchedEmails) {
// Find the original message's id (by serverId and mailbox)
Cursor c = getServerIdCursor(msg.mServerId, EmailContent.ID_PROJECTION);
@@ -762,10 +792,6 @@
if (id != null) {
userLog("Fetched body successfully for ", id);
final String[] bindArgument = new String[] {id};
- if ((maxPerFetch > 0) && (msg.mText.length() > maxPerFetch)) {
- userLog("Truncating message to " + maxPerFetch);
- msg.mText = msg.mText.substring(0, maxPerFetch) + "...";
- }
ops.add(ContentProviderOperation.newUpdate(EmailContent.Body.CONTENT_URI)
.withSelection(EmailContent.Body.MESSAGE_KEY + "=?", bindArgument)
.withValue(EmailContent.Body.TEXT_CONTENT, msg.mText)
@@ -776,16 +802,19 @@
EmailContent.Message.FLAG_LOADED_COMPLETE)
.build());
}
+ applyBatchIfNeeded(ops, maxOpsPerBatch, false);
}
for (EmailContent.Message msg: newEmails) {
msg.addSaveOps(ops);
+ applyBatchIfNeeded(ops, maxOpsPerBatch, false);
}
for (Long id : deletedEmails) {
ops.add(ContentProviderOperation.newDelete(
ContentUris.withAppendedId(EmailContent.Message.CONTENT_URI, id)).build());
AttachmentUtilities.deleteAllAttachmentFiles(mContext, mAccount.mId, id);
+ applyBatchIfNeeded(ops, maxOpsPerBatch, false);
}
if (!changedEmails.isEmpty()) {
@@ -806,6 +835,7 @@
.withValues(cv)
.build());
}
+ applyBatchIfNeeded(ops, maxOpsPerBatch, false);
}
// We only want to update the sync key here
@@ -815,16 +845,28 @@
ContentUris.withAppendedId(Mailbox.CONTENT_URI, mMailbox.mId))
.withValues(mailboxValues).build());
- try {
+ applyBatchIfNeeded(ops, maxOpsPerBatch, true);
+ userLog(mMailbox.mDisplayName, " SyncKey saved as: ", mMailbox.mSyncKey);
+ }
+
+ // Check if there at least MAX_OPS_PER_BATCH ops in queue and flush if there are.
+ // If force is true, flush regardless of size.
+ private void applyBatchIfNeeded(ArrayList<ContentProviderOperation> ops, int maxOpsPerBatch,
+ boolean force)
+ throws RemoteException, OperationApplicationException {
+ if (force || ops.size() >= maxOpsPerBatch) {
+ // STOPSHIP Remove calculating size of data before ship
+ if (LogUtils.isLoggable(TAG, Log.DEBUG)) {
+ final Parcel parcel = Parcel.obtain();
+ for (ContentProviderOperation op : ops) {
+ op.writeToParcel(parcel, 0);
+ }
+ Log.d(TAG, String.format("Committing %d ops total size=%d",
+ ops.size(), parcel.dataSize()));
+ parcel.recycle();
+ }
mContentResolver.applyBatch(EmailContent.AUTHORITY, ops);
- userLog(mMailbox.mDisplayName, " SyncKey saved as: ", mMailbox.mSyncKey);
- } catch (TransactionTooLargeException e) {
- LogUtils.w(TAG, "Transaction failed on fetched message; retrying...");
- commitImpl(++tryCount);
- } catch (RemoteException e) {
- // There is nothing to be done here; fail by returning null
- } catch (OperationApplicationException e) {
- // There is nothing to be done here; fail by returning null
+ ops.clear();
}
}
}
\ No newline at end of file
