platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogRefresherImpl.java - platform/tools/idea - Git at Google

 /*
  * Copyright 2000-2014 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package com.intellij.vcs.log.data;

 import com.intellij.openapi.diagnostic.Attachment;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.progress.ProgressIndicator;
 import com.intellij.openapi.progress.Task;
 import com.intellij.openapi.progress.impl.ProgressManagerImpl;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.util.Pair;
 import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.openapi.vcs.VcsException;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.util.Consumer;
 import com.intellij.util.Function;
 import com.intellij.util.NotNullFunction;
 import com.intellij.util.containers.ContainerUtil;
 import com.intellij.util.ui.UIUtil;
 import com.intellij.vcs.log.*;
 import com.intellij.vcs.log.graph.GraphCommit;
 import com.intellij.vcs.log.graph.GraphCommitImpl;
 import com.intellij.vcs.log.graph.PermanentGraph;
 import com.intellij.vcs.log.impl.RequirementsImpl;
 import com.intellij.vcs.log.util.StopWatch;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;

 import java.util.*;

 public class VcsLogRefresherImpl implements VcsLogRefresher {

   private static final Logger LOG = Logger.getInstance(VcsLogRefresherImpl.class);

   @NotNull private final Project myProject;
   @NotNull private final VcsLogHashMap myHashMap;
   @NotNull private final Map<VirtualFile, VcsLogProvider> myProviders;
   @NotNull private final VcsUserRegistryImpl myUserRegistry;
   @NotNull private final Map<Hash, VcsCommitMetadata> myTopCommitsDetailsCache;
   @NotNull private final Consumer<Exception> myExceptionHandler;
   private final int myRecentCommitCount;

   @NotNull private final SingleTaskController<RefreshRequest, DataPack> mySingleTaskController;

   @NotNull private DataPack myDataPack = EmptyDataPack.getInstance();

   public VcsLogRefresherImpl(@NotNull final Project project,
                              @NotNull VcsLogHashMap hashMap,
                              @NotNull Map<VirtualFile, VcsLogProvider> providers,
                              @NotNull final VcsUserRegistryImpl userRegistry,
                              @NotNull Map<Hash, VcsCommitMetadata> topCommitsDetailsCache,
                              @NotNull final Consumer<DataPack> dataPackUpdateHandler,
                              @NotNull Consumer<Exception> exceptionHandler,
                              int recentCommitsCount) {
     myProject = project;
     myHashMap = hashMap;
     myProviders = providers;
     myUserRegistry = userRegistry;
     myTopCommitsDetailsCache = topCommitsDetailsCache;
     myExceptionHandler = exceptionHandler;
     myRecentCommitCount = recentCommitsCount;

     Consumer<DataPack> dataPackUpdater = new Consumer<DataPack>() {
       @Override
       public void consume(@NotNull DataPack dataPack) {
         myDataPack = dataPack;
         dataPackUpdateHandler.consume(dataPack);
       }
     };
     mySingleTaskController = new SingleTaskController<RefreshRequest, DataPack>(dataPackUpdater) {
       @Override
       protected void startNewBackgroundTask() {
         UIUtil.invokeLaterIfNeeded(new Runnable() {
           @Override
           public void run() {
             ProgressManagerImpl.runProcessWithProgressAsynchronously(new MyRefreshTask(myDataPack));
           }
         });
       }
     };
   }

   @NotNull
   @Override
   public DataPack readFirstBlock() {
     try {
       LogInfo data = loadRecentData(new CommitCountRequirements(myRecentCommitCount).asMap(myProviders.keySet()));
       Collection<List<GraphCommit<Integer>>> commits = data.getCommits();
       Map<VirtualFile, Set<VcsRef>> refs = data.getRefs();
       DataPack dataPack = DataPack.build(multiRepoJoin(commits), refs, myProviders, myHashMap, false);
       mySingleTaskController.request(RefreshRequest.RELOAD_ALL); // build/rebuild the full log in background
       return dataPack;
     }
     catch (VcsException e) {
       myExceptionHandler.consume(e);
       return EmptyDataPack.getInstance();
     }
   }

   @NotNull
   private LogInfo loadRecentData(@NotNull final Map<VirtualFile, VcsLogProvider.Requirements> requirements) throws VcsException {
     final StopWatch sw = StopWatch.start("loading commits");
     final LogInfo logInfo = new LogInfo();
     new ProviderIterator() {
       @Override
       public void each(@NotNull VirtualFile root, @NotNull VcsLogProvider provider) throws VcsException {
         VcsLogProvider.DetailedLogData data = provider.readFirstBlock(root, requirements.get(root));
         storeUsersAndDetails(data.getCommits());
         logInfo.put(root, compactCommits(data.getCommits()));
         logInfo.put(root, data.getRefs());
         sw.rootCompleted(root);
       }
     }.iterate(getProvidersForRoots(requirements.keySet()));
     myUserRegistry.flush();
     sw.report();
     return logInfo;
   }

   @NotNull
   private Map<VirtualFile, VcsLogProvider> getProvidersForRoots(@NotNull Set<VirtualFile> roots) {
     return ContainerUtil.map2Map(roots,
                                  new Function<VirtualFile, Pair<VirtualFile, VcsLogProvider>>() {
                                    @Override
                                    public Pair<VirtualFile, VcsLogProvider> fun(VirtualFile root) {
                                      return Pair.create(root, myProviders.get(root));
                                    }
                                  });
   }

   @Override
   public void refresh(@NotNull Collection<VirtualFile> rootsToRefresh) {
     if (!rootsToRefresh.isEmpty()) {
       mySingleTaskController.request(new RefreshRequest(rootsToRefresh));
     }
   }

   @NotNull
   private static <T extends GraphCommit<Integer>> List<T> multiRepoJoin(@NotNull Collection<List<T>> commits) {
     StopWatch sw = StopWatch.start("multi-repo join");
     List<T> joined = new VcsLogMultiRepoJoiner<Integer, T>().join(commits);
     sw.report();
     return joined;
   }

   @NotNull
   private List<GraphCommit<Integer>> compactCommits(@NotNull List<? extends TimedVcsCommit> commits) {
     StopWatch sw = StopWatch.start("compacting commits");
     List<GraphCommit<Integer>> map = ContainerUtil.map(commits, new Function<TimedVcsCommit, GraphCommit<Integer>>() {
       @NotNull
       @Override
       public GraphCommit<Integer> fun(@NotNull TimedVcsCommit commit) {
         return compactCommit(commit);
       }
     });
     myHashMap.flush();
     sw.report();
     return map;
   }

   @NotNull
   private GraphCommitImpl<Integer> compactCommit(@NotNull TimedVcsCommit commit) {
     return new GraphCommitImpl<Integer>(myHashMap.getCommitIndex(commit.getId()),
                                         ContainerUtil.map(commit.getParents(), myHashMap.asIndexGetter()), commit.getTimestamp());
   }

   private void storeUsersAndDetails(@NotNull Collection<? extends VcsCommitMetadata> metadatas) {
     for (VcsCommitMetadata detail : metadatas) {
       myUserRegistry.addUser(detail.getAuthor());
       myUserRegistry.addUser(detail.getCommitter());
       myTopCommitsDetailsCache.put(detail.getId(), detail);
     }
   }

   private class MyRefreshTask extends Task.Backgroundable {

     @NotNull private DataPack myCurrentDataPack;

     @NotNull private final LogInfo myLoadedInfo = new LogInfo();

     MyRefreshTask(@NotNull DataPack currentDataPack) {
       super(VcsLogRefresherImpl.this.myProject, "Refreshing history...", false);
       myCurrentDataPack = currentDataPack;
     }

     @Override
     public void run(@NotNull ProgressIndicator indicator) {
       DataPack dataPack = myCurrentDataPack;
       while (true) {
         List<RefreshRequest> requests = mySingleTaskController.popRequests();
         Collection<VirtualFile> rootsToRefresh = getRootsToRefresh(requests);
         if (rootsToRefresh.isEmpty()) {
           mySingleTaskController.taskCompleted(dataPack);
           break;
         }
         dataPack = doRefresh(rootsToRefresh);
       }
     }

     @NotNull
     private Collection<VirtualFile> getRootsToRefresh(@NotNull List<RefreshRequest> requests) {
       Collection<VirtualFile> rootsToRefresh = ContainerUtil.newArrayList();
       for (RefreshRequest request : requests) {
         if (request == RefreshRequest.RELOAD_ALL) {
           myCurrentDataPack = EmptyDataPack.getInstance();
           return myProviders.keySet();
         }
         rootsToRefresh.addAll(request.rootsToRefresh);
       }
       return rootsToRefresh;
     }

     @NotNull
     private DataPack doRefresh(@NotNull Collection<VirtualFile> roots) {
       StopWatch sw = StopWatch.start("refresh");
       PermanentGraph<Integer> permanentGraph = myCurrentDataPack.isFull() ? myCurrentDataPack.getPermanentGraph() : null;
       Map<VirtualFile, Set<VcsRef>> currentRefs = myCurrentDataPack.getRefsModel().getAllRefsByRoot();
       try {
         if (permanentGraph != null) {
           int commitCount = myRecentCommitCount;
           for (int attempt = 0; attempt <= 1; attempt++) {
             loadLogAndRefs(roots, currentRefs, commitCount);
             List<? extends GraphCommit<Integer>> compoundLog = multiRepoJoin(myLoadedInfo.getCommits());
             Map<VirtualFile, Set<VcsRef>> allNewRefs = getAllNewRefs(myLoadedInfo, currentRefs);
             List<GraphCommit<Integer>> joinedFullLog = join(compoundLog, permanentGraph.getAllCommits(), currentRefs, allNewRefs);
             if (joinedFullLog == null) {
               commitCount *= 5;
             }
             else {
               return DataPack.build(joinedFullLog, allNewRefs, myProviders, myHashMap, true);
             }
           }
           // couldn't join => need to reload everything; if 5000 commits is still not enough, it's worth reporting:
           LOG.error("Couldn't join " + commitCount + " recent commits to the log (" + permanentGraph.getAllCommits().size() + " commits)",
                     new Attachment("recent_commits", myLoadedInfo.toLogString(myHashMap.asIndexGetter())));
         }

         return loadFullLog();
       }
       catch (Exception e) {
         myExceptionHandler.consume(e);
         return EmptyDataPack.getInstance();
       }
       finally {
         sw.report();
       }
     }

     @NotNull
     private Map<VirtualFile, Set<VcsRef>> getAllNewRefs(@NotNull LogInfo newInfo,
                                                         @NotNull Map<VirtualFile, Set<VcsRef>> previousRefs) {
       Map<VirtualFile, Set<VcsRef>> result = ContainerUtil.newHashMap();
       for (VirtualFile root : previousRefs.keySet()) {
         Set<VcsRef> newInfoRefs = newInfo.getRefs(root);
         result.put(root, newInfoRefs != null ? newInfoRefs : previousRefs.get(root));
       }
       return result;
     }

     private void loadLogAndRefs(@NotNull Collection<VirtualFile> roots,
                                 @NotNull Map<VirtualFile, Set<VcsRef>> prevRefs,
                                 int commitCount) throws VcsException {
       LogInfo logInfo = loadRecentData(prepareRequirements(roots, commitCount, prevRefs));
       for (VirtualFile root : roots) {
         myLoadedInfo.put(root, logInfo.getCommits(root));
         myLoadedInfo.put(root, logInfo.getRefs(root));
       }
     }

     @NotNull
     private Map<VirtualFile, VcsLogProvider.Requirements> prepareRequirements(@NotNull Collection<VirtualFile> roots,
                                                                               int commitCount,
                                                                               @NotNull Map<VirtualFile, Set<VcsRef>> prevRefs) {
       Map<VirtualFile, VcsLogProvider.Requirements> requirements = ContainerUtil.newHashMap();
       for (VirtualFile root : roots) {
         requirements.put(root, new RequirementsImpl(commitCount, true, ContainerUtil.notNullize(prevRefs.get(root))));
       }
       return requirements;
     }

     @Nullable
     private List<GraphCommit<Integer>> join(@NotNull List<? extends GraphCommit<Integer>> recentCommits,
                                             @NotNull List<GraphCommit<Integer>> fullLog,
                                             @NotNull Map<VirtualFile, Set<VcsRef>> previousRefs,
                                             @NotNull Map<VirtualFile, Set<VcsRef>> newRefs) {
       StopWatch sw = StopWatch.start("joining new commits");
       Function<VcsRef, Integer> ref2Int = new Function<VcsRef, Integer>() {
         @Override
         public Integer fun(@NotNull VcsRef ref) {
           return myHashMap.getCommitIndex(ref.getCommitHash());
         }
       };
       Collection<Integer> prevRefIndices = ContainerUtil.map(ContainerUtil.concat(previousRefs.values()), ref2Int);
       Collection<Integer> newRefIndices = ContainerUtil.map(ContainerUtil.concat(newRefs.values()), ref2Int);
       try {
         List<GraphCommit<Integer>> commits = new VcsLogJoiner<Integer, GraphCommit<Integer>>().addCommits(fullLog, prevRefIndices,
                                                                                                           recentCommits,
                                                                                                           newRefIndices).first;
         sw.report();
         return commits;
       }
       catch (VcsLogRefreshNotEnoughDataException e) {
         LOG.info(e);
       }
       catch (IllegalStateException e) {
         LOG.error(e);
       }
       return null;
     }

     @NotNull
     private DataPack loadFullLog() throws VcsException {
       StopWatch sw = StopWatch.start("full log reload");
       LogInfo logInfo = readFullLogFromVcs();
       List<? extends GraphCommit<Integer>> graphCommits = multiRepoJoin(logInfo.getCommits());
       DataPack dataPack = DataPack.build(graphCommits, logInfo.getRefs(), myProviders, myHashMap, true);
       sw.report();
       return dataPack;
     }

     @NotNull
     private LogInfo readFullLogFromVcs() throws VcsException {
       final StopWatch sw = StopWatch.start("read full log from VCS");
       final LogInfo logInfo = new LogInfo();
       new ProviderIterator() {
         @Override
         void each(@NotNull VirtualFile root, @NotNull VcsLogProvider provider) throws VcsException {
           final List<GraphCommit<Integer>> graphCommits = ContainerUtil.newArrayList();
           VcsLogProvider.LogData data = provider.readAllHashes(root, new Consumer<TimedVcsCommit>() {
             @Override
             public void consume(@NotNull TimedVcsCommit commit) {
               graphCommits.add(compactCommit(commit));
             }
           });
           logInfo.put(root, graphCommits);
           logInfo.put(root, data.getRefs());
           myUserRegistry.addUsers(data.getUsers());
           sw.rootCompleted(root);
         }
       }.iterate(myProviders);
       myUserRegistry.flush();
       sw.report();
       return logInfo;
     }
   }

   private static class RefreshRequest {
     private static final RefreshRequest RELOAD_ALL = new RefreshRequest(Collections.<VirtualFile>emptyList());
     private final Collection<VirtualFile> rootsToRefresh;

     RefreshRequest(@NotNull Collection<VirtualFile> rootsToRefresh) {
       this.rootsToRefresh = rootsToRefresh;
     }
   }

   private static abstract class ProviderIterator {
     abstract void each(@NotNull VirtualFile root, @NotNull VcsLogProvider provider) throws VcsException;

     final void iterate(@NotNull Map<VirtualFile, VcsLogProvider> providers) throws VcsException {
       for (Map.Entry<VirtualFile, VcsLogProvider> entry : providers.entrySet()) {
         each(entry.getKey(), entry.getValue());
       }
     }
   }

   private static class CommitCountRequirements implements VcsLogProvider.Requirements {
     private final int myCommitCount;

     public CommitCountRequirements(int commitCount) {
       myCommitCount = commitCount;
     }

     @Override
     public int getCommitCount() {
       return myCommitCount;
     }

     @NotNull
     Map<VirtualFile, VcsLogProvider.Requirements> asMap(@NotNull Collection<VirtualFile> roots) {
       return ContainerUtil.map2Map(roots, new Function<VirtualFile, Pair<VirtualFile, VcsLogProvider.Requirements>>() {
         @Override
         public Pair<VirtualFile, VcsLogProvider.Requirements> fun(VirtualFile root) {
           return Pair.<VirtualFile, VcsLogProvider.Requirements>create(root, CommitCountRequirements.this);
         }
       });
     }
   }

   private static class LogInfo {
     private final Map<VirtualFile, Set<VcsRef>> myRefs = ContainerUtil.newHashMap();
     private final Map<VirtualFile, List<GraphCommit<Integer>>> myCommits = ContainerUtil.newHashMap();

     void put(@NotNull VirtualFile root, @NotNull List<GraphCommit<Integer>> commits) {
       myCommits.put(root, commits);
     }

     void put(@NotNull VirtualFile root, @NotNull Set<VcsRef> refs) {
       myRefs.put(root, refs);
     }

     @NotNull
     Collection<List<GraphCommit<Integer>>> getCommits() {
       return myCommits.values();
     }

     List<GraphCommit<Integer>> getCommits(@NotNull VirtualFile root) {
       return myCommits.get(root);
     }

     @NotNull
     Map<VirtualFile, Set<VcsRef>> getRefs() {
       return myRefs;
     }

     public Set<VcsRef> getRefs(@NotNull VirtualFile root) {
       return myRefs.get(root);
     }

     @SuppressWarnings("StringConcatenationInsideStringBufferAppend")
     @NotNull
     public String toLogString(@NotNull final NotNullFunction<Hash, Integer> indexGetter) {
       StringBuilder sb = new StringBuilder();
       sb.append(" LOG:\n");
       for (Map.Entry<VirtualFile, List<GraphCommit<Integer>>> entry : myCommits.entrySet()) {
         sb.append(entry.getKey().getName() + "\n");
         sb.append(StringUtil.join(entry.getValue(), new Function<GraphCommit<Integer>, String>() {
           @Override
           public String fun(@NotNull GraphCommit<Integer> commit) {
             return commit.getId() + "<-" + StringUtil.join(commit.getParents(), ",");
           }
         }, "\n"));
       }
         sb.append("\nREFS:\n");
       for (Map.Entry<VirtualFile, Set<VcsRef>> entry : myRefs.entrySet()) {
         sb.append(entry.getKey().getName() + "\n");
         sb.append(StringUtil.join(entry.getValue(), new Function<VcsRef, String>() {
           @Override
           public String fun(@NotNull VcsRef ref) {
             return ref.getName() + "(" + indexGetter.fun(ref.getCommitHash()) + ")";
           }
         }, ","));
       }
       return sb.toString();
     }
   }
 }
	/*
	* Copyright 2000-2014 JetBrains s.r.o.
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package com.intellij.vcs.log.data;

	import com.intellij.openapi.diagnostic.Attachment;
	import com.intellij.openapi.diagnostic.Logger;
	import com.intellij.openapi.progress.ProgressIndicator;
	import com.intellij.openapi.progress.Task;
	import com.intellij.openapi.progress.impl.ProgressManagerImpl;
	import com.intellij.openapi.project.Project;
	import com.intellij.openapi.util.Pair;
	import com.intellij.openapi.util.text.StringUtil;
	import com.intellij.openapi.vcs.VcsException;
	import com.intellij.openapi.vfs.VirtualFile;
	import com.intellij.util.Consumer;
	import com.intellij.util.Function;
	import com.intellij.util.NotNullFunction;
	import com.intellij.util.containers.ContainerUtil;
	import com.intellij.util.ui.UIUtil;
	import com.intellij.vcs.log.*;
	import com.intellij.vcs.log.graph.GraphCommit;
	import com.intellij.vcs.log.graph.GraphCommitImpl;
	import com.intellij.vcs.log.graph.PermanentGraph;
	import com.intellij.vcs.log.impl.RequirementsImpl;
	import com.intellij.vcs.log.util.StopWatch;
	import org.jetbrains.annotations.NotNull;
	import org.jetbrains.annotations.Nullable;

	import java.util.*;

	public class VcsLogRefresherImpl implements VcsLogRefresher {

	private static final Logger LOG = Logger.getInstance(VcsLogRefresherImpl.class);

	@NotNull private final Project myProject;
	@NotNull private final VcsLogHashMap myHashMap;
	@NotNull private final Map<VirtualFile, VcsLogProvider> myProviders;
	@NotNull private final VcsUserRegistryImpl myUserRegistry;
	@NotNull private final Map<Hash, VcsCommitMetadata> myTopCommitsDetailsCache;
	@NotNull private final Consumer<Exception> myExceptionHandler;
	private final int myRecentCommitCount;

	@NotNull private final SingleTaskController<RefreshRequest, DataPack> mySingleTaskController;

	@NotNull private DataPack myDataPack = EmptyDataPack.getInstance();

	public VcsLogRefresherImpl(@NotNull final Project project,
	@NotNull VcsLogHashMap hashMap,
	@NotNull Map<VirtualFile, VcsLogProvider> providers,
	@NotNull final VcsUserRegistryImpl userRegistry,
	@NotNull Map<Hash, VcsCommitMetadata> topCommitsDetailsCache,
	@NotNull final Consumer<DataPack> dataPackUpdateHandler,
	@NotNull Consumer<Exception> exceptionHandler,
	int recentCommitsCount) {
	myProject = project;
	myHashMap = hashMap;
	myProviders = providers;
	myUserRegistry = userRegistry;
	myTopCommitsDetailsCache = topCommitsDetailsCache;
	myExceptionHandler = exceptionHandler;
	myRecentCommitCount = recentCommitsCount;

	Consumer<DataPack> dataPackUpdater = new Consumer<DataPack>() {
	@Override
	public void consume(@NotNull DataPack dataPack) {
	myDataPack = dataPack;
	dataPackUpdateHandler.consume(dataPack);
	}
	};
	mySingleTaskController = new SingleTaskController<RefreshRequest, DataPack>(dataPackUpdater) {
	@Override
	protected void startNewBackgroundTask() {
	UIUtil.invokeLaterIfNeeded(new Runnable() {
	@Override
	public void run() {
	ProgressManagerImpl.runProcessWithProgressAsynchronously(new MyRefreshTask(myDataPack));
	}
	});
	}
	};
	}

	@NotNull
	@Override
	public DataPack readFirstBlock() {
	try {
	LogInfo data = loadRecentData(new CommitCountRequirements(myRecentCommitCount).asMap(myProviders.keySet()));
	Collection<List<GraphCommit<Integer>>> commits = data.getCommits();
	Map<VirtualFile, Set<VcsRef>> refs = data.getRefs();
	DataPack dataPack = DataPack.build(multiRepoJoin(commits), refs, myProviders, myHashMap, false);
	mySingleTaskController.request(RefreshRequest.RELOAD_ALL); // build/rebuild the full log in background
	return dataPack;
	}
	catch (VcsException e) {
	myExceptionHandler.consume(e);
	return EmptyDataPack.getInstance();
	}
	}

	@NotNull
	private LogInfo loadRecentData(@NotNull final Map<VirtualFile, VcsLogProvider.Requirements> requirements) throws VcsException {
	final StopWatch sw = StopWatch.start("loading commits");
	final LogInfo logInfo = new LogInfo();
	new ProviderIterator() {
	@Override
	public void each(@NotNull VirtualFile root, @NotNull VcsLogProvider provider) throws VcsException {
	VcsLogProvider.DetailedLogData data = provider.readFirstBlock(root, requirements.get(root));
	storeUsersAndDetails(data.getCommits());
	logInfo.put(root, compactCommits(data.getCommits()));
	logInfo.put(root, data.getRefs());
	sw.rootCompleted(root);
	}
	}.iterate(getProvidersForRoots(requirements.keySet()));
	myUserRegistry.flush();
	sw.report();
	return logInfo;
	}

	@NotNull
	private Map<VirtualFile, VcsLogProvider> getProvidersForRoots(@NotNull Set<VirtualFile> roots) {
	return ContainerUtil.map2Map(roots,
	new Function<VirtualFile, Pair<VirtualFile, VcsLogProvider>>() {
	@Override
	public Pair<VirtualFile, VcsLogProvider> fun(VirtualFile root) {
	return Pair.create(root, myProviders.get(root));
	}
	});
	}

	@Override
	public void refresh(@NotNull Collection<VirtualFile> rootsToRefresh) {
	if (!rootsToRefresh.isEmpty()) {
	mySingleTaskController.request(new RefreshRequest(rootsToRefresh));
	}
	}

	@NotNull
	private static <T extends GraphCommit<Integer>> List<T> multiRepoJoin(@NotNull Collection<List<T>> commits) {
	StopWatch sw = StopWatch.start("multi-repo join");
	List<T> joined = new VcsLogMultiRepoJoiner<Integer, T>().join(commits);
	sw.report();
	return joined;
	}

	@NotNull
	private List<GraphCommit<Integer>> compactCommits(@NotNull List<? extends TimedVcsCommit> commits) {
	StopWatch sw = StopWatch.start("compacting commits");
	List<GraphCommit<Integer>> map = ContainerUtil.map(commits, new Function<TimedVcsCommit, GraphCommit<Integer>>() {
	@NotNull
	@Override
	public GraphCommit<Integer> fun(@NotNull TimedVcsCommit commit) {
	return compactCommit(commit);
	}
	});
	myHashMap.flush();
	sw.report();
	return map;
	}

	@NotNull
	private GraphCommitImpl<Integer> compactCommit(@NotNull TimedVcsCommit commit) {
	return new GraphCommitImpl<Integer>(myHashMap.getCommitIndex(commit.getId()),
	ContainerUtil.map(commit.getParents(), myHashMap.asIndexGetter()), commit.getTimestamp());
	}

	private void storeUsersAndDetails(@NotNull Collection<? extends VcsCommitMetadata> metadatas) {
	for (VcsCommitMetadata detail : metadatas) {
	myUserRegistry.addUser(detail.getAuthor());
	myUserRegistry.addUser(detail.getCommitter());
	myTopCommitsDetailsCache.put(detail.getId(), detail);
	}
	}

	private class MyRefreshTask extends Task.Backgroundable {

	@NotNull private DataPack myCurrentDataPack;

	@NotNull private final LogInfo myLoadedInfo = new LogInfo();

	MyRefreshTask(@NotNull DataPack currentDataPack) {
	super(VcsLogRefresherImpl.this.myProject, "Refreshing history...", false);
	myCurrentDataPack = currentDataPack;
	}

	@Override
	public void run(@NotNull ProgressIndicator indicator) {
	DataPack dataPack = myCurrentDataPack;
	while (true) {
	List<RefreshRequest> requests = mySingleTaskController.popRequests();
	Collection<VirtualFile> rootsToRefresh = getRootsToRefresh(requests);
	if (rootsToRefresh.isEmpty()) {
	mySingleTaskController.taskCompleted(dataPack);
	break;
	}
	dataPack = doRefresh(rootsToRefresh);
	}
	}

	@NotNull
	private Collection<VirtualFile> getRootsToRefresh(@NotNull List<RefreshRequest> requests) {
	Collection<VirtualFile> rootsToRefresh = ContainerUtil.newArrayList();
	for (RefreshRequest request : requests) {
	if (request == RefreshRequest.RELOAD_ALL) {
	myCurrentDataPack = EmptyDataPack.getInstance();
	return myProviders.keySet();
	}
	rootsToRefresh.addAll(request.rootsToRefresh);
	}
	return rootsToRefresh;
	}

	@NotNull
	private DataPack doRefresh(@NotNull Collection<VirtualFile> roots) {
	StopWatch sw = StopWatch.start("refresh");
	PermanentGraph<Integer> permanentGraph = myCurrentDataPack.isFull() ? myCurrentDataPack.getPermanentGraph() : null;
	Map<VirtualFile, Set<VcsRef>> currentRefs = myCurrentDataPack.getRefsModel().getAllRefsByRoot();
	try {
	if (permanentGraph != null) {
	int commitCount = myRecentCommitCount;
	for (int attempt = 0; attempt <= 1; attempt++) {
	loadLogAndRefs(roots, currentRefs, commitCount);
	List<? extends GraphCommit<Integer>> compoundLog = multiRepoJoin(myLoadedInfo.getCommits());
	Map<VirtualFile, Set<VcsRef>> allNewRefs = getAllNewRefs(myLoadedInfo, currentRefs);
	List<GraphCommit<Integer>> joinedFullLog = join(compoundLog, permanentGraph.getAllCommits(), currentRefs, allNewRefs);
	if (joinedFullLog == null) {
	commitCount *= 5;
	}
	else {
	return DataPack.build(joinedFullLog, allNewRefs, myProviders, myHashMap, true);
	}
	}
	// couldn't join => need to reload everything; if 5000 commits is still not enough, it's worth reporting:
	LOG.error("Couldn't join " + commitCount + " recent commits to the log (" + permanentGraph.getAllCommits().size() + " commits)",
	new Attachment("recent_commits", myLoadedInfo.toLogString(myHashMap.asIndexGetter())));
	}

	return loadFullLog();
	}
	catch (Exception e) {
	myExceptionHandler.consume(e);
	return EmptyDataPack.getInstance();
	}
	finally {
	sw.report();
	}
	}

	@NotNull
	private Map<VirtualFile, Set<VcsRef>> getAllNewRefs(@NotNull LogInfo newInfo,
	@NotNull Map<VirtualFile, Set<VcsRef>> previousRefs) {
	Map<VirtualFile, Set<VcsRef>> result = ContainerUtil.newHashMap();
	for (VirtualFile root : previousRefs.keySet()) {
	Set<VcsRef> newInfoRefs = newInfo.getRefs(root);
	result.put(root, newInfoRefs != null ? newInfoRefs : previousRefs.get(root));
	}
	return result;
	}

	private void loadLogAndRefs(@NotNull Collection<VirtualFile> roots,
	@NotNull Map<VirtualFile, Set<VcsRef>> prevRefs,
	int commitCount) throws VcsException {
	LogInfo logInfo = loadRecentData(prepareRequirements(roots, commitCount, prevRefs));
	for (VirtualFile root : roots) {
	myLoadedInfo.put(root, logInfo.getCommits(root));
	myLoadedInfo.put(root, logInfo.getRefs(root));
	}
	}

	@NotNull
	private Map<VirtualFile, VcsLogProvider.Requirements> prepareRequirements(@NotNull Collection<VirtualFile> roots,
	int commitCount,
	@NotNull Map<VirtualFile, Set<VcsRef>> prevRefs) {
	Map<VirtualFile, VcsLogProvider.Requirements> requirements = ContainerUtil.newHashMap();
	for (VirtualFile root : roots) {
	requirements.put(root, new RequirementsImpl(commitCount, true, ContainerUtil.notNullize(prevRefs.get(root))));
	}
	return requirements;
	}

	@Nullable
	private List<GraphCommit<Integer>> join(@NotNull List<? extends GraphCommit<Integer>> recentCommits,
	@NotNull List<GraphCommit<Integer>> fullLog,
	@NotNull Map<VirtualFile, Set<VcsRef>> previousRefs,
	@NotNull Map<VirtualFile, Set<VcsRef>> newRefs) {
	StopWatch sw = StopWatch.start("joining new commits");
	Function<VcsRef, Integer> ref2Int = new Function<VcsRef, Integer>() {
	@Override
	public Integer fun(@NotNull VcsRef ref) {
	return myHashMap.getCommitIndex(ref.getCommitHash());
	}
	};
	Collection<Integer> prevRefIndices = ContainerUtil.map(ContainerUtil.concat(previousRefs.values()), ref2Int);
	Collection<Integer> newRefIndices = ContainerUtil.map(ContainerUtil.concat(newRefs.values()), ref2Int);
	try {
	List<GraphCommit<Integer>> commits = new VcsLogJoiner<Integer, GraphCommit<Integer>>().addCommits(fullLog, prevRefIndices,
	recentCommits,
	newRefIndices).first;
	sw.report();
	return commits;
	}
	catch (VcsLogRefreshNotEnoughDataException e) {
	LOG.info(e);
	}
	catch (IllegalStateException e) {
	LOG.error(e);
	}
	return null;
	}

	@NotNull
	private DataPack loadFullLog() throws VcsException {
	StopWatch sw = StopWatch.start("full log reload");
	LogInfo logInfo = readFullLogFromVcs();
	List<? extends GraphCommit<Integer>> graphCommits = multiRepoJoin(logInfo.getCommits());
	DataPack dataPack = DataPack.build(graphCommits, logInfo.getRefs(), myProviders, myHashMap, true);
	sw.report();
	return dataPack;
	}

	@NotNull
	private LogInfo readFullLogFromVcs() throws VcsException {
	final StopWatch sw = StopWatch.start("read full log from VCS");
	final LogInfo logInfo = new LogInfo();
	new ProviderIterator() {
	@Override
	void each(@NotNull VirtualFile root, @NotNull VcsLogProvider provider) throws VcsException {
	final List<GraphCommit<Integer>> graphCommits = ContainerUtil.newArrayList();
	VcsLogProvider.LogData data = provider.readAllHashes(root, new Consumer<TimedVcsCommit>() {
	@Override
	public void consume(@NotNull TimedVcsCommit commit) {
	graphCommits.add(compactCommit(commit));
	}
	});
	logInfo.put(root, graphCommits);
	logInfo.put(root, data.getRefs());
	myUserRegistry.addUsers(data.getUsers());
	sw.rootCompleted(root);
	}
	}.iterate(myProviders);
	myUserRegistry.flush();
	sw.report();
	return logInfo;
	}
	}

	private static class RefreshRequest {
	private static final RefreshRequest RELOAD_ALL = new RefreshRequest(Collections.<VirtualFile>emptyList());
	private final Collection<VirtualFile> rootsToRefresh;

	RefreshRequest(@NotNull Collection<VirtualFile> rootsToRefresh) {
	this.rootsToRefresh = rootsToRefresh;
	}
	}

	private static abstract class ProviderIterator {
	abstract void each(@NotNull VirtualFile root, @NotNull VcsLogProvider provider) throws VcsException;

	final void iterate(@NotNull Map<VirtualFile, VcsLogProvider> providers) throws VcsException {
	for (Map.Entry<VirtualFile, VcsLogProvider> entry : providers.entrySet()) {
	each(entry.getKey(), entry.getValue());
	}
	}
	}

	private static class CommitCountRequirements implements VcsLogProvider.Requirements {
	private final int myCommitCount;

	public CommitCountRequirements(int commitCount) {
	myCommitCount = commitCount;
	}

	@Override
	public int getCommitCount() {
	return myCommitCount;
	}

	@NotNull
	Map<VirtualFile, VcsLogProvider.Requirements> asMap(@NotNull Collection<VirtualFile> roots) {
	return ContainerUtil.map2Map(roots, new Function<VirtualFile, Pair<VirtualFile, VcsLogProvider.Requirements>>() {
	@Override
	public Pair<VirtualFile, VcsLogProvider.Requirements> fun(VirtualFile root) {
	return Pair.<VirtualFile, VcsLogProvider.Requirements>create(root, CommitCountRequirements.this);
	}
	});
	}
	}

	private static class LogInfo {
	private final Map<VirtualFile, Set<VcsRef>> myRefs = ContainerUtil.newHashMap();
	private final Map<VirtualFile, List<GraphCommit<Integer>>> myCommits = ContainerUtil.newHashMap();

	void put(@NotNull VirtualFile root, @NotNull List<GraphCommit<Integer>> commits) {
	myCommits.put(root, commits);
	}

	void put(@NotNull VirtualFile root, @NotNull Set<VcsRef> refs) {
	myRefs.put(root, refs);
	}

	@NotNull
	Collection<List<GraphCommit<Integer>>> getCommits() {
	return myCommits.values();
	}

	List<GraphCommit<Integer>> getCommits(@NotNull VirtualFile root) {
	return myCommits.get(root);
	}

	@NotNull
	Map<VirtualFile, Set<VcsRef>> getRefs() {
	return myRefs;
	}

	public Set<VcsRef> getRefs(@NotNull VirtualFile root) {
	return myRefs.get(root);
	}

	@SuppressWarnings("StringConcatenationInsideStringBufferAppend")
	@NotNull
	public String toLogString(@NotNull final NotNullFunction<Hash, Integer> indexGetter) {
	StringBuilder sb = new StringBuilder();
	sb.append(" LOG:\n");
	for (Map.Entry<VirtualFile, List<GraphCommit<Integer>>> entry : myCommits.entrySet()) {
	sb.append(entry.getKey().getName() + "\n");
	sb.append(StringUtil.join(entry.getValue(), new Function<GraphCommit<Integer>, String>() {
	@Override
	public String fun(@NotNull GraphCommit<Integer> commit) {
	return commit.getId() + "<-" + StringUtil.join(commit.getParents(), ",");
	}
	}, "\n"));
	}
	sb.append("\nREFS:\n");
	for (Map.Entry<VirtualFile, Set<VcsRef>> entry : myRefs.entrySet()) {
	sb.append(entry.getKey().getName() + "\n");
	sb.append(StringUtil.join(entry.getValue(), new Function<VcsRef, String>() {
	@Override
	public String fun(@NotNull VcsRef ref) {
	return ref.getName() + "(" + indexGetter.fun(ref.getCommitHash()) + ")";
	}
	}, ","));
	}
	return sb.toString();
	}
	}
	}