plugins/maven/src/main/java/org/jetbrains/idea/maven/indices/MavenIndex.java - platform/tools/idea - Git at Google

 /*
  * Copyright 2000-2014 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.jetbrains.idea.maven.indices;

 import com.intellij.openapi.util.ThrowableComputable;
 import com.intellij.openapi.util.io.FileUtil;
 import com.intellij.util.io.*;
 import gnu.trove.THashMap;
 import gnu.trove.THashSet;
 import org.apache.lucene.search.Query;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jetbrains.annotations.TestOnly;
 import org.jetbrains.idea.maven.model.MavenArtifactInfo;
 import org.jetbrains.idea.maven.model.MavenId;
 import org.jetbrains.idea.maven.project.MavenGeneralSettings;
 import org.jetbrains.idea.maven.server.MavenIndexerWrapper;
 import org.jetbrains.idea.maven.server.MavenIndicesProcessor;
 import org.jetbrains.idea.maven.server.MavenServerIndexerException;
 import org.jetbrains.idea.maven.utils.MavenLog;
 import org.jetbrains.idea.maven.utils.MavenProcessCanceledException;
 import org.jetbrains.idea.maven.utils.MavenProgressIndicator;

 import java.io.*;
 import java.util.*;

 public class MavenIndex {
   private static final String CURRENT_VERSION = "4";

   protected static final String INDEX_INFO_FILE = "index.properties";

   private static final String INDEX_VERSION_KEY = "version";
   private static final String KIND_KEY = "kind";
   private static final String ID_KEY = "id";
   private static final String PATH_OR_URL_KEY = "pathOrUrl";
   private static final String TIMESTAMP_KEY = "lastUpdate";
   private static final String DATA_DIR_NAME_KEY = "dataDirName";
   private static final String FAILURE_MESSAGE_KEY = "failureMessage";

   private static final String UPDATE_DIR = "update";

   private static final String DATA_DIR_PREFIX = "data";

   private static final String ARTIFACT_IDS_MAP_FILE = "artifactIds-map.dat";
   private static final String VERSIONS_MAP_FILE = "versions-map.dat";

   public enum Kind {
     LOCAL, REMOTE
   }

   private final MavenIndexerWrapper myIndexer;
   private final File myDir;

   private final String myRepositoryId;
   private final String myRepositoryPathOrUrl;
   private final Kind myKind;
   private Long myUpdateTimestamp;

   private String myDataDirName;
   private IndexData myData;

   private String myFailureMessage;

   private boolean isBroken;
   private final IndexListener myListener;

   public MavenIndex(MavenIndexerWrapper indexer,
                     File dir,
                     String repositoryId,
                     String repositoryPathOrUrl,
                     Kind kind,
                     IndexListener listener) throws MavenIndexException {
     myIndexer = indexer;
     myDir = dir;
     myRepositoryId = repositoryId;
     myRepositoryPathOrUrl = normalizePathOrUrl(repositoryPathOrUrl);
     myKind = kind;
     myListener = listener;

     open();
   }

   public MavenIndex(MavenIndexerWrapper indexer,
                     File dir,
                     IndexListener listener) throws MavenIndexException {
     myIndexer = indexer;
     myDir = dir;
     myListener = listener;

     Properties props = new Properties();
     try {
       FileInputStream s = new FileInputStream(new File(dir, INDEX_INFO_FILE));
       try {
         props.load(s);
       }
       finally {
         s.close();
       }
     }
     catch (IOException e) {
       throw new MavenIndexException("Cannot read " + INDEX_INFO_FILE + " file", e);
     }

     if (!CURRENT_VERSION.equals(props.getProperty(INDEX_VERSION_KEY))) {
       throw new MavenIndexException("Incompatible index version, needs to be updated: " + dir);
     }

     myKind = Kind.valueOf(props.getProperty(KIND_KEY));
     myRepositoryId = props.getProperty(ID_KEY);
     myRepositoryPathOrUrl = normalizePathOrUrl(props.getProperty(PATH_OR_URL_KEY));

     try {
       String timestamp = props.getProperty(TIMESTAMP_KEY);
       if (timestamp != null) myUpdateTimestamp = Long.parseLong(timestamp);
     }
     catch (Exception ignored) {
     }

     myDataDirName = props.getProperty(DATA_DIR_NAME_KEY);
     myFailureMessage = props.getProperty(FAILURE_MESSAGE_KEY);

     if (!getUpdateDir().exists()) {
       myUpdateTimestamp = null;
     }

     open();
   }

   private static String normalizePathOrUrl(String pathOrUrl) {
     pathOrUrl = pathOrUrl.trim();
     pathOrUrl = FileUtil.toSystemIndependentName(pathOrUrl);
     while (pathOrUrl.endsWith("/")) {
       pathOrUrl = pathOrUrl.substring(0, pathOrUrl.length() - 1);
     }
     return pathOrUrl;
   }

   private void open() throws MavenIndexException {
     try {
       try {
         doOpen();
       }
       catch (Exception e1) {
         final boolean versionUpdated = e1.getCause() instanceof PersistentEnumeratorBase.VersionUpdatedException;
         if (!versionUpdated) MavenLog.LOG.warn(e1);

         try {
           doOpen();
         }
         catch (Exception e2) {
           throw new MavenIndexException("Cannot open index " + myDir.getPath(), e2);
         }
         markAsBroken();
       }
     }
     finally {
       save();
     }
   }

   private void doOpen() throws Exception {
     try {
       File dataDir;
       if (myDataDirName == null) {
         dataDir = createNewDataDir();
         myDataDirName = dataDir.getName();
       }
       else {
         dataDir = new File(myDir, myDataDirName);
         dataDir.mkdirs();
       }
       myData = new IndexData(dataDir);
     }
     catch (Exception e) {
       cleanupBrokenData();
       throw e;
     }
   }

   private void cleanupBrokenData() {
     close(true);
     FileUtil.delete(getCurrentDataDir());
     myDataDirName = null;
   }

   public synchronized void close(boolean releaseIndexContext) {
     try {
       if (myData != null) myData.close(releaseIndexContext);
     }
     catch (MavenIndexException e) {
       MavenLog.LOG.warn(e);
     }
     myData = null;
   }

   private synchronized void save() {
     myDir.mkdirs();

     Properties props = new Properties();

     props.setProperty(KIND_KEY, myKind.toString());
     props.setProperty(ID_KEY, myRepositoryId);
     props.setProperty(PATH_OR_URL_KEY, myRepositoryPathOrUrl);
     props.setProperty(INDEX_VERSION_KEY, CURRENT_VERSION);
     if (myUpdateTimestamp != null) props.setProperty(TIMESTAMP_KEY, String.valueOf(myUpdateTimestamp));
     if (myDataDirName != null) props.setProperty(DATA_DIR_NAME_KEY, myDataDirName);
     if (myFailureMessage != null) props.setProperty(FAILURE_MESSAGE_KEY, myFailureMessage);

     try {
       FileOutputStream s = new FileOutputStream(new File(myDir, INDEX_INFO_FILE));
       try {
         props.store(s, null);
       }
       finally {
         s.close();
       }
     }
     catch (IOException e) {
       MavenLog.LOG.warn(e);
     }
   }

   public String getRepositoryId() {
     return myRepositoryId;
   }

   public File getRepositoryFile() {
     return myKind == Kind.LOCAL ? new File(myRepositoryPathOrUrl) : null;
   }

   public String getRepositoryUrl() {
     return myKind == Kind.REMOTE ? myRepositoryPathOrUrl : null;
   }

   public String getRepositoryPathOrUrl() {
     return myRepositoryPathOrUrl;
   }

   public Kind getKind() {
     return myKind;
   }

   public boolean isFor(Kind kind, String repositoryId, String pathOrUrl) {
     if (myKind != kind || !myRepositoryId.equals(repositoryId)) return false;
     if (kind == Kind.LOCAL) return FileUtil.pathsEqual(myRepositoryPathOrUrl, normalizePathOrUrl(pathOrUrl));
     return myRepositoryPathOrUrl.equalsIgnoreCase(normalizePathOrUrl(pathOrUrl));
   }

   public synchronized long getUpdateTimestamp() {
     return myUpdateTimestamp == null ? -1 : myUpdateTimestamp;
   }

   public synchronized String getFailureMessage() {
     return myFailureMessage;
   }

   public void updateOrRepair(boolean fullUpdate, MavenGeneralSettings settings, MavenProgressIndicator progress)
     throws MavenProcessCanceledException {
     try {
       if (fullUpdate) {
         if (myKind == Kind.LOCAL) FileUtil.delete(getUpdateDir());
         int context = createContext(getUpdateDir(), "update");
         try {
           updateContext(context, settings, progress);
         }
         finally {
           myIndexer.releaseIndex(context);
         }
       }
       updateData(progress);

       isBroken = false;
       myFailureMessage = null;
     }
     catch (MavenProcessCanceledException e) {
       throw e;
     }
     catch (Exception e) {
       handleUpdateException(e);
     }

     save();
   }

   private void handleUpdateException(Exception e) {
     myFailureMessage = e.getMessage();
     MavenLog.LOG.warn("Failed to update Maven indices for: [" + myRepositoryId + "] " + myRepositoryPathOrUrl, e);
   }

   private int createContext(File contextDir, String suffix) throws MavenServerIndexerException {
     String indexId = myDir.getName() + "-" + suffix;
     return myIndexer.createIndex(indexId,
                                  myRepositoryId,
                                  getRepositoryFile(),
                                  getRepositoryUrl(),
                                  contextDir);
   }

   private File getUpdateDir() {
     return new File(myDir, UPDATE_DIR);
   }

   private void updateContext(int indexId, MavenGeneralSettings settings, MavenProgressIndicator progress)
     throws MavenServerIndexerException, MavenProcessCanceledException {
     myIndexer.updateIndex(indexId, settings, progress);
   }

   private void updateData(MavenProgressIndicator progress) throws MavenIndexException {
     IndexData newData;

     File newDataDir = createNewDataDir();
     try {
       FileUtil.copyDir(getUpdateDir(), getDataContextDir(newDataDir));
     }
     catch (IOException e) {
       throw new MavenIndexException(e);
     }
     newData = new IndexData(newDataDir);

     try {
       doUpdateIndexData(newData, progress);
       newData.flush();
     }
     catch (Throwable e) {
       newData.close(true);
       FileUtil.delete(newDataDir);

       if (e instanceof MavenServerIndexerException) throw new MavenIndexException(e);
       if (e instanceof IOException) throw new MavenIndexException(e);
       throw new RuntimeException(e);
     }

     synchronized (this) {
       IndexData oldData = myData;

       myData = newData;
       myDataDirName = newDataDir.getName();

       myUpdateTimestamp = System.currentTimeMillis();

       oldData.close(true);

       for (File each : FileUtil.notNullize(myDir.listFiles())) {
         if (each.getName().startsWith(DATA_DIR_PREFIX) && !each.getName().equals(myDataDirName)) {
           FileUtil.delete(each);
         }
       }
     }
   }

   private void doUpdateIndexData(IndexData data,
                                  MavenProgressIndicator progress) throws IOException, MavenServerIndexerException {
     final Map<String, Set<String>> groupToArtifactMap = new THashMap<String, Set<String>>();
     final Map<String, Set<String>> groupWithArtifactToVersionMap = new THashMap<String, Set<String>>();

     final StringBuilder builder = new StringBuilder();

     progress.pushState();
     progress.setIndeterminate(true);

     try {
       myIndexer.processArtifacts(data.indexId, new MavenIndicesProcessor() {
         @Override
         public void processArtifacts(Collection<MavenId> artifacts) {
           for (MavenId each : artifacts) {
             String groupId = each.getGroupId();
             String artifactId = each.getArtifactId();
             String version = each.getVersion();

             builder.setLength(0);

             builder.append(groupId).append(":").append(artifactId);
             String ga = builder.toString();

             getOrCreate(groupToArtifactMap, groupId).add(artifactId);
             getOrCreate(groupWithArtifactToVersionMap, ga).add(version);
           }
         }
       });

       persist(groupToArtifactMap, data.groupToArtifactMap);
       persist(groupWithArtifactToVersionMap, data.groupWithArtifactToVersionMap);
     }
     finally {
       progress.popState();
     }
   }

   private static <T> Set<T> getOrCreate(Map<String, Set<T>> map, String key) {
     Set<T> result = map.get(key);
     if (result == null) {
       result = new THashSet<T>();
       map.put(key, result);
     }
     return result;
   }

   private static <T> void persist(Map<String, T> map, PersistentHashMap<String, T> persistentMap) throws IOException {
     for (Map.Entry<String, T> each : map.entrySet()) {
       persistentMap.put(each.getKey(), each.getValue());
     }
   }

   @TestOnly
   public File getDir() {
     return myDir;
   }

   @TestOnly
   protected synchronized File getCurrentDataDir() {
     return new File(myDir, myDataDirName);
   }

   private static File getDataContextDir(File dataDir) {
     return new File(dataDir, "context");
   }

   @NotNull
   private File createNewDataDir() {
     return MavenIndices.createNewDir(myDir, DATA_DIR_PREFIX, 100);
   }

   public synchronized void addArtifact(final File artifactFile) {
     doIndexTask(new IndexTask<Object>() {
       public Object doTask() throws Exception {
         MavenId id = myData.addArtifact(artifactFile);

         String groupId = id.getGroupId();
         String artifactId = id.getArtifactId();
         String version = id.getVersion();

         myData.hasGroupCache.put(groupId, true);

         String groupWithArtifact = groupId + ":" + artifactId;

         myData.hasArtifactCache.put(groupWithArtifact, true);
         myData.hasVersionCache.put(groupWithArtifact + ':' + version, true);

         addToCache(myData.groupToArtifactMap, groupId, artifactId);
         addToCache(myData.groupWithArtifactToVersionMap, groupWithArtifact, version);
         myData.flush();

         return null;
       }
     }, null);
   }

   private static void addToCache(PersistentHashMap<String, Set<String>> cache, String key, String value) throws IOException {
     Set<String> values = cache.get(key);
     if (values == null) values = new THashSet<String>();
     values.add(value);
     cache.put(key, values);
   }

   public synchronized Collection<String> getGroupIds() {
     return doIndexTask(new IndexTask<Collection<String>>() {
       public Collection<String> doTask() throws Exception {
         return myData.groupToArtifactMap.getAllDataObjects(null);
       }
     }, Collections.<String>emptySet());
   }

   public synchronized Set<String> getArtifactIds(final String groupId) {
     return doIndexTask(new IndexTask<Set<String>>() {
       public Set<String> doTask() throws Exception {
         Set<String> result = myData.groupToArtifactMap.get(groupId);
         return result == null ? Collections.<String>emptySet() : result;
       }
     }, Collections.<String>emptySet());
   }

   @TestOnly
   public synchronized void printInfo() {
     doIndexTask(new IndexTask<Set<String>>() {
       public Set<String> doTask() throws Exception {
         System.out.println("BaseFile: " + myData.groupToArtifactMap.getBaseFile());
         System.out.println("All data objects: " + myData.groupToArtifactMap.getAllDataObjects(null));
         return Collections.<String>emptySet();
       }
     }, Collections.<String>emptySet());
   }

   public synchronized Set<String> getVersions(final String groupId, final String artifactId) {
     return doIndexTask(new IndexTask<Set<String>>() {
       public Set<String> doTask() throws Exception {
         Set<String> result = myData.groupWithArtifactToVersionMap.get(groupId + ":" + artifactId);
         return result == null ? Collections.<String>emptySet() : result;
       }
     }, Collections.<String>emptySet());
   }

   public synchronized boolean hasGroupId(String groupId) {
     if (isBroken) return false;

     return hasValue(myData.groupToArtifactMap, myData.hasGroupCache, groupId);
   }

   public synchronized boolean hasArtifactId(String groupId, String artifactId) {
     if (isBroken) return false;

     return hasValue(myData.groupWithArtifactToVersionMap, myData.hasArtifactCache, groupId + ":" + artifactId);
   }

   public synchronized boolean hasVersion(String groupId, String artifactId, final String version) {
     if (isBroken) return false;

     final String groupWithArtifactWithVersion = groupId + ":" + artifactId + ':' + version;

     Boolean res = myData.hasVersionCache.get(groupWithArtifactWithVersion);
     if (res == null) {
       res = doIndexTask(new IndexTask<Boolean>() {
         @Override
         public Boolean doTask() throws Exception {
           String groupWithVersion = groupWithArtifactWithVersion.substring(0, groupWithArtifactWithVersion.length() - version.length() - 1);
           Set<String> set = myData.groupWithArtifactToVersionMap.get(groupWithVersion);
           return set != null && set.contains(version);
         }
       }, false);

       myData.hasVersionCache.put(groupWithArtifactWithVersion, res);
     }

     return res;
   }

   private boolean hasValue(final PersistentHashMap<String, ?> map, Map<String, Boolean> cache, final String value) {
     Boolean res = cache.get(value);
     if (res == null) {
       res = doIndexTask(new IndexTask<Boolean>() {
         public Boolean doTask() throws Exception {
           return map.tryEnumerate(value) != 0;
         }
       }, false).booleanValue();

       cache.put(value, res);
     }

     return res;
   }

   public synchronized Set<MavenArtifactInfo> search(final Query query, final int maxResult) {
     return doIndexTask(new IndexTask<Set<MavenArtifactInfo>>() {
       public Set<MavenArtifactInfo> doTask() throws Exception {
         return myData.search(query, maxResult);
       }
     }, Collections.<MavenArtifactInfo>emptySet());
   }

   private <T> T doIndexTask(IndexTask<T> task, T defaultValue) {
     assert Thread.holdsLock(this);

     if (!isBroken) {
       try {
         return task.doTask();
       }
       catch (Exception e1) {
         MavenLog.LOG.warn(e1);

         cleanupBrokenData();
         try {
           open();
         }
         catch (MavenIndexException e2) {
           MavenLog.LOG.warn(e2);
         }
       }
     }
     markAsBroken();
     return defaultValue;
   }

   private void markAsBroken() {
     if (!isBroken) {
       myListener.indexIsBroken(this);
     }
     isBroken = true;
   }

   private interface IndexTask<T> {
     T doTask() throws Exception;
   }

   private class IndexData {
     final PersistentHashMap<String, Set<String>> groupToArtifactMap;
     final PersistentHashMap<String, Set<String>> groupWithArtifactToVersionMap;

     final Map<String, Boolean> hasGroupCache = new THashMap<String, Boolean>();
     final Map<String, Boolean> hasArtifactCache = new THashMap<String, Boolean>();
     final Map<String, Boolean> hasVersionCache = new THashMap<String, Boolean>();

     private final int indexId;

     public IndexData(File dir) throws MavenIndexException {
       try {
         groupToArtifactMap = createPersistentMap(new File(dir, ARTIFACT_IDS_MAP_FILE));
         groupWithArtifactToVersionMap = createPersistentMap(new File(dir, VERSIONS_MAP_FILE));

         indexId = createContext(getDataContextDir(dir), dir.getName());
       }
       catch (IOException e) {
         close(true);
         throw new MavenIndexException(e);
       }
       catch (MavenServerIndexerException e) {
         close(true);
         throw new MavenIndexException(e);
       }
     }

     private PersistentHashMap<String, Set<String>> createPersistentMap(final File f) throws IOException {
       return new PersistentHashMap<String, Set<String>>(f, new EnumeratorStringDescriptor(), new SetDescriptor());
     }

     public void close(boolean releaseIndexContext) throws MavenIndexException {
       MavenIndexException[] exceptions = new MavenIndexException[1];

       try {
         if (indexId != 0 && releaseIndexContext) myIndexer.releaseIndex(indexId);
       }
       catch (MavenServerIndexerException e) {
         MavenLog.LOG.warn(e);
         if (exceptions[0] == null) exceptions[0] = new MavenIndexException(e);
       }

       safeClose(groupToArtifactMap, exceptions);
       safeClose(groupWithArtifactToVersionMap, exceptions);

       if (exceptions[0] != null) throw exceptions[0];
     }

     private void safeClose(@Nullable Closeable enumerator, MavenIndexException[] exceptions) {
       try {
         if (enumerator != null) enumerator.close();
       }
       catch (IOException e) {
         MavenLog.LOG.warn(e);
         if (exceptions[0] == null) exceptions[0] = new MavenIndexException(e);
       }
     }

     public void flush() throws IOException {
       groupToArtifactMap.force();
       groupWithArtifactToVersionMap.force();
     }

     public MavenId addArtifact(File artifactFile) throws MavenServerIndexerException {
       return myIndexer.addArtifact(indexId, artifactFile);
     }

     public Set<MavenArtifactInfo> search(Query query, int maxResult) throws MavenServerIndexerException {
       return myIndexer.search(indexId, query, maxResult);
     }
   }

   private static class SetDescriptor implements DataExternalizer<Set<String>> {
     public void save(@NotNull DataOutput s, Set<String> set) throws IOException {
       s.writeInt(set.size());
       for (String each : set) {
         s.writeUTF(each);
       }
     }

     public Set<String> read(@NotNull DataInput s) throws IOException {
       int count = s.readInt();
       Set<String> result = new THashSet<String>(count);
       while (count-- > 0) {
         result.add(s.readUTF());
       }
       return result;
     }
   }

   public interface IndexListener {
     void indexIsBroken(MavenIndex index);
   }
 }
	/*
	* Copyright 2000-2014 JetBrains s.r.o.
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package org.jetbrains.idea.maven.indices;

	import com.intellij.openapi.util.ThrowableComputable;
	import com.intellij.openapi.util.io.FileUtil;
	import com.intellij.util.io.*;
	import gnu.trove.THashMap;
	import gnu.trove.THashSet;
	import org.apache.lucene.search.Query;
	import org.jetbrains.annotations.NotNull;
	import org.jetbrains.annotations.Nullable;
	import org.jetbrains.annotations.TestOnly;
	import org.jetbrains.idea.maven.model.MavenArtifactInfo;
	import org.jetbrains.idea.maven.model.MavenId;
	import org.jetbrains.idea.maven.project.MavenGeneralSettings;
	import org.jetbrains.idea.maven.server.MavenIndexerWrapper;
	import org.jetbrains.idea.maven.server.MavenIndicesProcessor;
	import org.jetbrains.idea.maven.server.MavenServerIndexerException;
	import org.jetbrains.idea.maven.utils.MavenLog;
	import org.jetbrains.idea.maven.utils.MavenProcessCanceledException;
	import org.jetbrains.idea.maven.utils.MavenProgressIndicator;

	import java.io.*;
	import java.util.*;

	public class MavenIndex {
	private static final String CURRENT_VERSION = "4";

	protected static final String INDEX_INFO_FILE = "index.properties";

	private static final String INDEX_VERSION_KEY = "version";
	private static final String KIND_KEY = "kind";
	private static final String ID_KEY = "id";
	private static final String PATH_OR_URL_KEY = "pathOrUrl";
	private static final String TIMESTAMP_KEY = "lastUpdate";
	private static final String DATA_DIR_NAME_KEY = "dataDirName";
	private static final String FAILURE_MESSAGE_KEY = "failureMessage";

	private static final String UPDATE_DIR = "update";

	private static final String DATA_DIR_PREFIX = "data";

	private static final String ARTIFACT_IDS_MAP_FILE = "artifactIds-map.dat";
	private static final String VERSIONS_MAP_FILE = "versions-map.dat";

	public enum Kind {
	LOCAL, REMOTE
	}

	private final MavenIndexerWrapper myIndexer;
	private final File myDir;

	private final String myRepositoryId;
	private final String myRepositoryPathOrUrl;
	private final Kind myKind;
	private Long myUpdateTimestamp;

	private String myDataDirName;
	private IndexData myData;

	private String myFailureMessage;

	private boolean isBroken;
	private final IndexListener myListener;

	public MavenIndex(MavenIndexerWrapper indexer,
	File dir,
	String repositoryId,
	String repositoryPathOrUrl,
	Kind kind,
	IndexListener listener) throws MavenIndexException {
	myIndexer = indexer;
	myDir = dir;
	myRepositoryId = repositoryId;
	myRepositoryPathOrUrl = normalizePathOrUrl(repositoryPathOrUrl);
	myKind = kind;
	myListener = listener;

	open();
	}

	public MavenIndex(MavenIndexerWrapper indexer,
	File dir,
	IndexListener listener) throws MavenIndexException {
	myIndexer = indexer;
	myDir = dir;
	myListener = listener;

	Properties props = new Properties();
	try {
	FileInputStream s = new FileInputStream(new File(dir, INDEX_INFO_FILE));
	try {
	props.load(s);
	}
	finally {
	s.close();
	}
	}
	catch (IOException e) {
	throw new MavenIndexException("Cannot read " + INDEX_INFO_FILE + " file", e);
	}

	if (!CURRENT_VERSION.equals(props.getProperty(INDEX_VERSION_KEY))) {
	throw new MavenIndexException("Incompatible index version, needs to be updated: " + dir);
	}

	myKind = Kind.valueOf(props.getProperty(KIND_KEY));
	myRepositoryId = props.getProperty(ID_KEY);
	myRepositoryPathOrUrl = normalizePathOrUrl(props.getProperty(PATH_OR_URL_KEY));

	try {
	String timestamp = props.getProperty(TIMESTAMP_KEY);
	if (timestamp != null) myUpdateTimestamp = Long.parseLong(timestamp);
	}
	catch (Exception ignored) {
	}

	myDataDirName = props.getProperty(DATA_DIR_NAME_KEY);
	myFailureMessage = props.getProperty(FAILURE_MESSAGE_KEY);

	if (!getUpdateDir().exists()) {
	myUpdateTimestamp = null;
	}

	open();
	}

	private static String normalizePathOrUrl(String pathOrUrl) {
	pathOrUrl = pathOrUrl.trim();
	pathOrUrl = FileUtil.toSystemIndependentName(pathOrUrl);
	while (pathOrUrl.endsWith("/")) {
	pathOrUrl = pathOrUrl.substring(0, pathOrUrl.length() - 1);
	}
	return pathOrUrl;
	}

	private void open() throws MavenIndexException {
	try {
	try {
	doOpen();
	}
	catch (Exception e1) {
	final boolean versionUpdated = e1.getCause() instanceof PersistentEnumeratorBase.VersionUpdatedException;
	if (!versionUpdated) MavenLog.LOG.warn(e1);

	try {
	doOpen();
	}
	catch (Exception e2) {
	throw new MavenIndexException("Cannot open index " + myDir.getPath(), e2);
	}
	markAsBroken();
	}
	}
	finally {
	save();
	}
	}

	private void doOpen() throws Exception {
	try {
	File dataDir;
	if (myDataDirName == null) {
	dataDir = createNewDataDir();
	myDataDirName = dataDir.getName();
	}
	else {
	dataDir = new File(myDir, myDataDirName);
	dataDir.mkdirs();
	}
	myData = new IndexData(dataDir);
	}
	catch (Exception e) {
	cleanupBrokenData();
	throw e;
	}
	}

	private void cleanupBrokenData() {
	close(true);
	FileUtil.delete(getCurrentDataDir());
	myDataDirName = null;
	}

	public synchronized void close(boolean releaseIndexContext) {
	try {
	if (myData != null) myData.close(releaseIndexContext);
	}
	catch (MavenIndexException e) {
	MavenLog.LOG.warn(e);
	}
	myData = null;
	}

	private synchronized void save() {
	myDir.mkdirs();

	Properties props = new Properties();

	props.setProperty(KIND_KEY, myKind.toString());
	props.setProperty(ID_KEY, myRepositoryId);
	props.setProperty(PATH_OR_URL_KEY, myRepositoryPathOrUrl);
	props.setProperty(INDEX_VERSION_KEY, CURRENT_VERSION);
	if (myUpdateTimestamp != null) props.setProperty(TIMESTAMP_KEY, String.valueOf(myUpdateTimestamp));
	if (myDataDirName != null) props.setProperty(DATA_DIR_NAME_KEY, myDataDirName);
	if (myFailureMessage != null) props.setProperty(FAILURE_MESSAGE_KEY, myFailureMessage);

	try {
	FileOutputStream s = new FileOutputStream(new File(myDir, INDEX_INFO_FILE));
	try {
	props.store(s, null);
	}
	finally {
	s.close();
	}
	}
	catch (IOException e) {
	MavenLog.LOG.warn(e);
	}
	}

	public String getRepositoryId() {
	return myRepositoryId;
	}

	public File getRepositoryFile() {
	return myKind == Kind.LOCAL ? new File(myRepositoryPathOrUrl) : null;
	}

	public String getRepositoryUrl() {
	return myKind == Kind.REMOTE ? myRepositoryPathOrUrl : null;
	}

	public String getRepositoryPathOrUrl() {
	return myRepositoryPathOrUrl;
	}

	public Kind getKind() {
	return myKind;
	}

	public boolean isFor(Kind kind, String repositoryId, String pathOrUrl) {
	if (myKind != kind \|\| !myRepositoryId.equals(repositoryId)) return false;
	if (kind == Kind.LOCAL) return FileUtil.pathsEqual(myRepositoryPathOrUrl, normalizePathOrUrl(pathOrUrl));
	return myRepositoryPathOrUrl.equalsIgnoreCase(normalizePathOrUrl(pathOrUrl));
	}

	public synchronized long getUpdateTimestamp() {
	return myUpdateTimestamp == null ? -1 : myUpdateTimestamp;
	}

	public synchronized String getFailureMessage() {
	return myFailureMessage;
	}

	public void updateOrRepair(boolean fullUpdate, MavenGeneralSettings settings, MavenProgressIndicator progress)
	throws MavenProcessCanceledException {
	try {
	if (fullUpdate) {
	if (myKind == Kind.LOCAL) FileUtil.delete(getUpdateDir());
	int context = createContext(getUpdateDir(), "update");
	try {
	updateContext(context, settings, progress);
	}
	finally {
	myIndexer.releaseIndex(context);
	}
	}
	updateData(progress);

	isBroken = false;
	myFailureMessage = null;
	}
	catch (MavenProcessCanceledException e) {
	throw e;
	}
	catch (Exception e) {
	handleUpdateException(e);
	}

	save();
	}

	private void handleUpdateException(Exception e) {
	myFailureMessage = e.getMessage();
	MavenLog.LOG.warn("Failed to update Maven indices for: [" + myRepositoryId + "] " + myRepositoryPathOrUrl, e);
	}

	private int createContext(File contextDir, String suffix) throws MavenServerIndexerException {
	String indexId = myDir.getName() + "-" + suffix;
	return myIndexer.createIndex(indexId,
	myRepositoryId,
	getRepositoryFile(),
	getRepositoryUrl(),
	contextDir);
	}

	private File getUpdateDir() {
	return new File(myDir, UPDATE_DIR);
	}

	private void updateContext(int indexId, MavenGeneralSettings settings, MavenProgressIndicator progress)
	throws MavenServerIndexerException, MavenProcessCanceledException {
	myIndexer.updateIndex(indexId, settings, progress);
	}

	private void updateData(MavenProgressIndicator progress) throws MavenIndexException {
	IndexData newData;

	File newDataDir = createNewDataDir();
	try {
	FileUtil.copyDir(getUpdateDir(), getDataContextDir(newDataDir));
	}
	catch (IOException e) {
	throw new MavenIndexException(e);
	}
	newData = new IndexData(newDataDir);

	try {
	doUpdateIndexData(newData, progress);
	newData.flush();
	}
	catch (Throwable e) {
	newData.close(true);
	FileUtil.delete(newDataDir);

	if (e instanceof MavenServerIndexerException) throw new MavenIndexException(e);
	if (e instanceof IOException) throw new MavenIndexException(e);
	throw new RuntimeException(e);
	}

	synchronized (this) {
	IndexData oldData = myData;

	myData = newData;
	myDataDirName = newDataDir.getName();

	myUpdateTimestamp = System.currentTimeMillis();

	oldData.close(true);

	for (File each : FileUtil.notNullize(myDir.listFiles())) {
	if (each.getName().startsWith(DATA_DIR_PREFIX) && !each.getName().equals(myDataDirName)) {
	FileUtil.delete(each);
	}
	}
	}
	}

	private void doUpdateIndexData(IndexData data,
	MavenProgressIndicator progress) throws IOException, MavenServerIndexerException {
	final Map<String, Set<String>> groupToArtifactMap = new THashMap<String, Set<String>>();
	final Map<String, Set<String>> groupWithArtifactToVersionMap = new THashMap<String, Set<String>>();

	final StringBuilder builder = new StringBuilder();

	progress.pushState();
	progress.setIndeterminate(true);

	try {
	myIndexer.processArtifacts(data.indexId, new MavenIndicesProcessor() {
	@Override
	public void processArtifacts(Collection<MavenId> artifacts) {
	for (MavenId each : artifacts) {
	String groupId = each.getGroupId();
	String artifactId = each.getArtifactId();
	String version = each.getVersion();

	builder.setLength(0);

	builder.append(groupId).append(":").append(artifactId);
	String ga = builder.toString();

	getOrCreate(groupToArtifactMap, groupId).add(artifactId);
	getOrCreate(groupWithArtifactToVersionMap, ga).add(version);
	}
	}
	});

	persist(groupToArtifactMap, data.groupToArtifactMap);
	persist(groupWithArtifactToVersionMap, data.groupWithArtifactToVersionMap);
	}
	finally {
	progress.popState();
	}
	}

	private static <T> Set<T> getOrCreate(Map<String, Set<T>> map, String key) {
	Set<T> result = map.get(key);
	if (result == null) {
	result = new THashSet<T>();
	map.put(key, result);
	}
	return result;
	}

	private static <T> void persist(Map<String, T> map, PersistentHashMap<String, T> persistentMap) throws IOException {
	for (Map.Entry<String, T> each : map.entrySet()) {
	persistentMap.put(each.getKey(), each.getValue());
	}
	}

	@TestOnly
	public File getDir() {
	return myDir;
	}

	@TestOnly
	protected synchronized File getCurrentDataDir() {
	return new File(myDir, myDataDirName);
	}

	private static File getDataContextDir(File dataDir) {
	return new File(dataDir, "context");
	}

	@NotNull
	private File createNewDataDir() {
	return MavenIndices.createNewDir(myDir, DATA_DIR_PREFIX, 100);
	}

	public synchronized void addArtifact(final File artifactFile) {
	doIndexTask(new IndexTask<Object>() {
	public Object doTask() throws Exception {
	MavenId id = myData.addArtifact(artifactFile);

	String groupId = id.getGroupId();
	String artifactId = id.getArtifactId();
	String version = id.getVersion();

	myData.hasGroupCache.put(groupId, true);

	String groupWithArtifact = groupId + ":" + artifactId;

	myData.hasArtifactCache.put(groupWithArtifact, true);
	myData.hasVersionCache.put(groupWithArtifact + ':' + version, true);

	addToCache(myData.groupToArtifactMap, groupId, artifactId);
	addToCache(myData.groupWithArtifactToVersionMap, groupWithArtifact, version);
	myData.flush();

	return null;
	}
	}, null);
	}

	private static void addToCache(PersistentHashMap<String, Set<String>> cache, String key, String value) throws IOException {
	Set<String> values = cache.get(key);
	if (values == null) values = new THashSet<String>();
	values.add(value);
	cache.put(key, values);
	}

	public synchronized Collection<String> getGroupIds() {
	return doIndexTask(new IndexTask<Collection<String>>() {
	public Collection<String> doTask() throws Exception {
	return myData.groupToArtifactMap.getAllDataObjects(null);
	}
	}, Collections.<String>emptySet());
	}

	public synchronized Set<String> getArtifactIds(final String groupId) {
	return doIndexTask(new IndexTask<Set<String>>() {
	public Set<String> doTask() throws Exception {
	Set<String> result = myData.groupToArtifactMap.get(groupId);
	return result == null ? Collections.<String>emptySet() : result;
	}
	}, Collections.<String>emptySet());
	}

	@TestOnly
	public synchronized void printInfo() {
	doIndexTask(new IndexTask<Set<String>>() {
	public Set<String> doTask() throws Exception {
	System.out.println("BaseFile: " + myData.groupToArtifactMap.getBaseFile());
	System.out.println("All data objects: " + myData.groupToArtifactMap.getAllDataObjects(null));
	return Collections.<String>emptySet();
	}
	}, Collections.<String>emptySet());
	}

	public synchronized Set<String> getVersions(final String groupId, final String artifactId) {
	return doIndexTask(new IndexTask<Set<String>>() {
	public Set<String> doTask() throws Exception {
	Set<String> result = myData.groupWithArtifactToVersionMap.get(groupId + ":" + artifactId);
	return result == null ? Collections.<String>emptySet() : result;
	}
	}, Collections.<String>emptySet());
	}

	public synchronized boolean hasGroupId(String groupId) {
	if (isBroken) return false;

	return hasValue(myData.groupToArtifactMap, myData.hasGroupCache, groupId);
	}

	public synchronized boolean hasArtifactId(String groupId, String artifactId) {
	if (isBroken) return false;

	return hasValue(myData.groupWithArtifactToVersionMap, myData.hasArtifactCache, groupId + ":" + artifactId);
	}

	public synchronized boolean hasVersion(String groupId, String artifactId, final String version) {
	if (isBroken) return false;

	final String groupWithArtifactWithVersion = groupId + ":" + artifactId + ':' + version;

	Boolean res = myData.hasVersionCache.get(groupWithArtifactWithVersion);
	if (res == null) {
	res = doIndexTask(new IndexTask<Boolean>() {
	@Override
	public Boolean doTask() throws Exception {
	String groupWithVersion = groupWithArtifactWithVersion.substring(0, groupWithArtifactWithVersion.length() - version.length() - 1);
	Set<String> set = myData.groupWithArtifactToVersionMap.get(groupWithVersion);
	return set != null && set.contains(version);
	}
	}, false);

	myData.hasVersionCache.put(groupWithArtifactWithVersion, res);
	}

	return res;
	}

	private boolean hasValue(final PersistentHashMap<String, ?> map, Map<String, Boolean> cache, final String value) {
	Boolean res = cache.get(value);
	if (res == null) {
	res = doIndexTask(new IndexTask<Boolean>() {
	public Boolean doTask() throws Exception {
	return map.tryEnumerate(value) != 0;
	}
	}, false).booleanValue();

	cache.put(value, res);
	}

	return res;
	}

	public synchronized Set<MavenArtifactInfo> search(final Query query, final int maxResult) {
	return doIndexTask(new IndexTask<Set<MavenArtifactInfo>>() {
	public Set<MavenArtifactInfo> doTask() throws Exception {
	return myData.search(query, maxResult);
	}
	}, Collections.<MavenArtifactInfo>emptySet());
	}

	private <T> T doIndexTask(IndexTask<T> task, T defaultValue) {
	assert Thread.holdsLock(this);

	if (!isBroken) {
	try {
	return task.doTask();
	}
	catch (Exception e1) {
	MavenLog.LOG.warn(e1);

	cleanupBrokenData();
	try {
	open();
	}
	catch (MavenIndexException e2) {
	MavenLog.LOG.warn(e2);
	}
	}
	}
	markAsBroken();
	return defaultValue;
	}

	private void markAsBroken() {
	if (!isBroken) {
	myListener.indexIsBroken(this);
	}
	isBroken = true;
	}

	private interface IndexTask<T> {
	T doTask() throws Exception;
	}

	private class IndexData {
	final PersistentHashMap<String, Set<String>> groupToArtifactMap;
	final PersistentHashMap<String, Set<String>> groupWithArtifactToVersionMap;

	final Map<String, Boolean> hasGroupCache = new THashMap<String, Boolean>();
	final Map<String, Boolean> hasArtifactCache = new THashMap<String, Boolean>();
	final Map<String, Boolean> hasVersionCache = new THashMap<String, Boolean>();

	private final int indexId;

	public IndexData(File dir) throws MavenIndexException {
	try {
	groupToArtifactMap = createPersistentMap(new File(dir, ARTIFACT_IDS_MAP_FILE));
	groupWithArtifactToVersionMap = createPersistentMap(new File(dir, VERSIONS_MAP_FILE));

	indexId = createContext(getDataContextDir(dir), dir.getName());
	}
	catch (IOException e) {
	close(true);
	throw new MavenIndexException(e);
	}
	catch (MavenServerIndexerException e) {
	close(true);
	throw new MavenIndexException(e);
	}
	}

	private PersistentHashMap<String, Set<String>> createPersistentMap(final File f) throws IOException {
	return new PersistentHashMap<String, Set<String>>(f, new EnumeratorStringDescriptor(), new SetDescriptor());
	}

	public void close(boolean releaseIndexContext) throws MavenIndexException {
	MavenIndexException[] exceptions = new MavenIndexException[1];

	try {
	if (indexId != 0 && releaseIndexContext) myIndexer.releaseIndex(indexId);
	}
	catch (MavenServerIndexerException e) {
	MavenLog.LOG.warn(e);
	if (exceptions[0] == null) exceptions[0] = new MavenIndexException(e);
	}

	safeClose(groupToArtifactMap, exceptions);
	safeClose(groupWithArtifactToVersionMap, exceptions);

	if (exceptions[0] != null) throw exceptions[0];
	}

	private void safeClose(@Nullable Closeable enumerator, MavenIndexException[] exceptions) {
	try {
	if (enumerator != null) enumerator.close();
	}
	catch (IOException e) {
	MavenLog.LOG.warn(e);
	if (exceptions[0] == null) exceptions[0] = new MavenIndexException(e);
	}
	}

	public void flush() throws IOException {
	groupToArtifactMap.force();
	groupWithArtifactToVersionMap.force();
	}

	public MavenId addArtifact(File artifactFile) throws MavenServerIndexerException {
	return myIndexer.addArtifact(indexId, artifactFile);
	}

	public Set<MavenArtifactInfo> search(Query query, int maxResult) throws MavenServerIndexerException {
	return myIndexer.search(indexId, query, maxResult);
	}
	}

	private static class SetDescriptor implements DataExternalizer<Set<String>> {
	public void save(@NotNull DataOutput s, Set<String> set) throws IOException {
	s.writeInt(set.size());
	for (String each : set) {
	s.writeUTF(each);
	}
	}

	public Set<String> read(@NotNull DataInput s) throws IOException {
	int count = s.readInt();
	Set<String> result = new THashSet<String>(count);
	while (count-- > 0) {
	result.add(s.readUTF());
	}
	return result;
	}
	}

	public interface IndexListener {
	void indexIsBroken(MavenIndex index);
	}
	}