platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java - platform/tools/idea - Git at Google

 /*
  * Copyright 2000-2014 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 package com.intellij.util.indexing;

 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.progress.ProgressManager;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.util.io.FileUtil;
 import com.intellij.psi.search.GlobalSearchScope;
 import com.intellij.psi.search.ProjectAndLibrariesScope;
 import com.intellij.psi.search.ProjectScopeImpl;
 import com.intellij.util.CommonProcessors;
 import com.intellij.util.Processor;
 import com.intellij.util.SystemProperties;
 import com.intellij.util.containers.SLRUCache;
 import com.intellij.util.io.*;
 import com.intellij.util.io.DataOutputStream;
 import gnu.trove.TIntHashSet;
 import gnu.trove.TIntProcedure;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;

 import java.io.*;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;

 /**
  * @author Eugene Zhuravlev
  *         Date: Dec 20, 2007
  */
 public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Value>{
   private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapIndexStorage");
   private static final boolean ENABLE_CACHED_HASH_IDS = SystemProperties.getBooleanProperty("idea.index.no.cashed.hashids", true);
   private final boolean myBuildKeyHashToVirtualFileMapping;
   private PersistentMap<Key, ValueContainer<Value>> myMap;
   private PersistentBTreeEnumerator<int[]> myKeyHashToVirtualFileMapping;
   private SLRUCache<Key, ChangeTrackingValueContainer<Value>> myCache;
   private volatile int myLastScannedId;
   private final File myStorageFile;
   private final KeyDescriptor<Key> myKeyDescriptor;
   private final int myCacheSize;

   private final Lock l = new ReentrantLock();
   private final DataExternalizer<Value> myDataExternalizer;
   private final boolean myKeyIsUniqueForIndexedFile;

   public MapIndexStorage(@NotNull File storageFile,
                          @NotNull KeyDescriptor<Key> keyDescriptor,
                          @NotNull DataExternalizer<Value> valueExternalizer,
                          final int cacheSize
   ) throws IOException {
     this(storageFile, keyDescriptor, valueExternalizer, cacheSize, false, false);
   }

   public MapIndexStorage(@NotNull File storageFile,
                          @NotNull KeyDescriptor<Key> keyDescriptor,
                          @NotNull DataExternalizer<Value> valueExternalizer,
                          final int cacheSize,
                          boolean keyIsUniqueForIndexedFile,
                          boolean buildKeyHashToVirtualFileMapping) throws IOException {
     myStorageFile = storageFile;
     myKeyDescriptor = keyDescriptor;
     myCacheSize = cacheSize;
     myDataExternalizer = valueExternalizer;
     myKeyIsUniqueForIndexedFile = keyIsUniqueForIndexedFile;
     myBuildKeyHashToVirtualFileMapping = buildKeyHashToVirtualFileMapping && FileBasedIndex.ourEnableTracingOfKeyHashToVirtualFileMapping;
     initMapAndCache();
   }

   private void initMapAndCache() throws IOException {
     final ValueContainerMap<Key, Value> map = new ValueContainerMap<Key, Value>(myStorageFile, myKeyDescriptor, myDataExternalizer,
                                                                                 myKeyIsUniqueForIndexedFile);
     myCache = new SLRUCache<Key, ChangeTrackingValueContainer<Value>>(myCacheSize, (int)(Math.ceil(myCacheSize * 0.25)) /* 25% from the main cache size*/) {
       @Override
       @NotNull
       public ChangeTrackingValueContainer<Value> createValue(final Key key) {
         return new ChangeTrackingValueContainer<Value>(new ChangeTrackingValueContainer.Initializer<Value>() {
           @NotNull
           @Override
           public Object getLock() {
             return map.getDataAccessLock();
           }

           @Nullable
           @Override
           public ValueContainer<Value> compute() {
             ValueContainer<Value> value;
             try {
               value = map.get(key);
               if (value == null) {
                 value = new ValueContainerImpl<Value>();
               }
             }
             catch (IOException e) {
               throw new RuntimeException(e);
             }
             return value;
           }
         });
       }

       @Override
       protected void onDropFromCache(final Key key, @NotNull final ChangeTrackingValueContainer<Value> valueContainer) {
         if (valueContainer.isDirty()) {
           try {
             map.put(key, valueContainer);
           }
           catch (IOException e) {
             throw new RuntimeException(e);
           }
         }
       }
     };

     myMap = map;

     myKeyHashToVirtualFileMapping = myBuildKeyHashToVirtualFileMapping ? new KeyHash2VirtualFileEnumerator(getProjectFile()) : null;
   }

   @NotNull
   private File getProjectFile() {
     return new File(myStorageFile.getPath() + ".project");
   }

   @Override
   public void flush() {
     l.lock();
     try {
       if (!myMap.isClosed()) {
         myCache.clear();
         if (myMap.isDirty()) myMap.force();
       }
       if (myKeyHashToVirtualFileMapping != null && myKeyHashToVirtualFileMapping.isDirty()) myKeyHashToVirtualFileMapping.force();
     }
     finally {
       l.unlock();
     }
   }

   @Override
   public void close() throws StorageException {
     try {
       flush();
       if (myKeyHashToVirtualFileMapping != null) myKeyHashToVirtualFileMapping.close();
       myMap.close();
     }
     catch (IOException e) {
       throw new StorageException(e);
     }
     catch (RuntimeException e) {
       final Throwable cause = e.getCause();
       if (cause instanceof IOException) {
         throw new StorageException(cause);
       }
       if (cause instanceof StorageException) {
         throw (StorageException)cause;
       }
       throw e;
     }
   }

   @Override
   public void clear() throws StorageException{
     try {
       myMap.close();
       if (myKeyHashToVirtualFileMapping != null) myKeyHashToVirtualFileMapping.close();
     }
     catch (IOException e) {
       LOG.error(e);
     }
     try {
       FileUtil.delete(myStorageFile);
       if (myKeyHashToVirtualFileMapping != null) IOUtil.deleteAllFilesStartingWith(getProjectFile());
       initMapAndCache();
     }
     catch (IOException e) {
       throw new StorageException(e);
     }
     catch (RuntimeException e) {
       final Throwable cause = e.getCause();
       if (cause instanceof IOException) {
         throw new StorageException(cause);
       }
       if (cause instanceof StorageException) {
         throw (StorageException)cause;
       }
       throw e;
     }
   }

   @Override
   public boolean processKeys(@NotNull final Processor<Key> processor, GlobalSearchScope scope, final IdFilter idFilter) throws StorageException {
     l.lock();
     try {
       myCache.clear(); // this will ensure that all new keys are made into the map
       if (myBuildKeyHashToVirtualFileMapping && idFilter != null) {
         TIntHashSet hashMaskSet = null;
         long l = System.currentTimeMillis();

         File fileWithCaches = getSavedProjectFileValueIds(myLastScannedId, scope);
         final boolean useCachedHashIds = ENABLE_CACHED_HASH_IDS &&
                                          (scope instanceof ProjectScopeImpl || scope instanceof ProjectAndLibrariesScope) &&
                                          fileWithCaches != null;
         int id = myKeyHashToVirtualFileMapping.getLargestId();

         if (useCachedHashIds && id == myLastScannedId) {
           try {
             hashMaskSet = loadHashedIds(fileWithCaches);
           } catch (IOException ignored) {
           }
         }

         if (hashMaskSet == null) {
           if (useCachedHashIds && myLastScannedId != 0) {
             FileUtil.asyncDelete(fileWithCaches);
           }

           hashMaskSet = new TIntHashSet(1000);
           final TIntHashSet finalHashMaskSet = hashMaskSet;
           myKeyHashToVirtualFileMapping.iterateData(new Processor<int[]>() {
             @Override
             public boolean process(int[] key) {
               if (!idFilter.containsFileId(key[1])) return true;
               finalHashMaskSet.add(key[0]);
               ProgressManager.checkCanceled();
               return true;
             }
           });

           if (useCachedHashIds) {
             saveHashedIds(hashMaskSet, id, scope);
           }
         }

         if (LOG.isDebugEnabled()) {
           LOG.debug("Scanned keyHashToVirtualFileMapping of " + myStorageFile + " for " + (System.currentTimeMillis() - l));
         }
         final TIntHashSet finalHashMaskSet = hashMaskSet;
         return myMap.processKeys(new Processor<Key>() {
           @Override
           public boolean process(Key key) {
             if (!finalHashMaskSet.contains(myKeyDescriptor.getHashCode(key))) return true;
             return processor.process(key);
           }
         });
       }
       return myMap.processKeys(processor);
     }
     catch (IOException e) {
       throw new StorageException(e);
     }
     catch (RuntimeException e) {
       final Throwable cause = e.getCause();
       if (cause instanceof IOException) {
         throw new StorageException(cause);
       }
       if (cause instanceof StorageException) {
         throw (StorageException)cause;
       }
       throw e;
     }
     finally {
       l.unlock();
     }
   }

   @NotNull
   private static TIntHashSet loadHashedIds(@NotNull File fileWithCaches) throws IOException {
     DataInputStream inputStream = null;
     try {
       inputStream = new DataInputStream(new BufferedInputStream(new FileInputStream(fileWithCaches)));
       int capacity = DataInputOutputUtil.readINT(inputStream);
       TIntHashSet hashMaskSet = new TIntHashSet(capacity);
       while(capacity > 0) {
         hashMaskSet.add(DataInputOutputUtil.readINT(inputStream));
         --capacity;
       }
       inputStream.close();
       return hashMaskSet;
     }
     finally {
       if (inputStream != null) {
         try {
           inputStream.close();
         }
         catch (IOException ignored) {}
       }
     }
   }

   private void saveHashedIds(@NotNull TIntHashSet hashMaskSet, int largestId, @NotNull GlobalSearchScope scope) {
     File newFileWithCaches = getSavedProjectFileValueIds(largestId, scope);
     assert newFileWithCaches != null;
     DataOutputStream stream = null;

     boolean savedSuccessfully = false;
     try {
       stream = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(newFileWithCaches)));
       DataInputOutputUtil.writeINT(stream, hashMaskSet.size());
       final DataOutputStream finalStream = stream;
       savedSuccessfully = hashMaskSet.forEach(new TIntProcedure() {
         @Override
         public boolean execute(int value) {
           try {
             DataInputOutputUtil.writeINT(finalStream, value);
             return true;
           } catch (IOException ex) {
             return false;
           }
         }
       });
     }
     catch (IOException ignored) {
     }
     finally {
       if (stream != null) {
         try {
           stream.close();
           if (savedSuccessfully) myLastScannedId = largestId;
         }
         catch (IOException ignored) {}
       }
     }
   }

   @Nullable
   private File getSavedProjectFileValueIds(int id, @NotNull GlobalSearchScope scope) {
     Project project = scope.getProject();
     if (project == null) return null;
     return new File(myStorageFile.getPath() + ".project."+project.hashCode() + "."+id + "." + scope.isSearchInLibraries());
   }

   @NotNull
   @Override
   public Collection<Key> getKeys() throws StorageException {
     List<Key> keys = new ArrayList<Key>();
     processKeys(new CommonProcessors.CollectProcessor<Key>(keys), null, null);
     return keys;
   }

   @Override
   @NotNull
   public ChangeTrackingValueContainer<Value> read(final Key key) throws StorageException {
     l.lock();
     try {
       return myCache.get(key);
     }
     catch (RuntimeException e) {
       final Throwable cause = e.getCause();
       if (cause instanceof IOException) {
         throw new StorageException(cause);
       }
       if (cause instanceof StorageException) {
         throw (StorageException)cause;
       }
       throw e;
     }
     finally {
       l.unlock();
     }
   }

   @Override
   public void addValue(final Key key, final int inputId, final Value value) throws StorageException {
     try {
       if (myKeyHashToVirtualFileMapping != null) {
         myKeyHashToVirtualFileMapping.enumerate(new int[] { myKeyDescriptor.getHashCode(key), inputId });
       }

       myMap.markDirty();
       if (!myKeyIsUniqueForIndexedFile) {
         read(key).addValue(inputId, value);
         return;
       }

       ChangeTrackingValueContainer<Value> cached;
       try {
         l.lock();
         cached = myCache.getIfCached(key);
       } finally {
         l.unlock();
       }

       if (cached != null) {
         cached.addValue(inputId, value);
         return;
       }
       // do not pollute the cache with keys unique to indexed file
       ChangeTrackingValueContainer<Value> valueContainer = new ChangeTrackingValueContainer<Value>(null);
       valueContainer.addValue(inputId, value);
       myMap.put(key, valueContainer);
     }
     catch (IOException e) {
       throw new StorageException(e);
     }
   }

   @Override
   public void removeAllValues(@NotNull Key key, int inputId) throws StorageException {
     try {
       myMap.markDirty();
       // important: assuming the key exists in the index
       read(key).removeAssociatedValue(inputId);
     }
     catch (IOException e) {
       throw new StorageException(e);
     }
   }

   private static class IntPairInArrayKeyDescriptor implements KeyDescriptor<int[]>, DifferentSerializableBytesImplyNonEqualityPolicy {
     @Override
     public void save(@NotNull DataOutput out, int[] value) throws IOException {
       DataInputOutputUtil.writeINT(out, value[0]);
       DataInputOutputUtil.writeINT(out, value[1]);
     }

     @Override
     public int[] read(@NotNull DataInput in) throws IOException {
       return new int[] {DataInputOutputUtil.readINT(in), DataInputOutputUtil.readINT(in)};
     }

     @Override
     public int getHashCode(int[] value) {
       return value[0] * 31 + value[1];
     }

     @Override
     public boolean isEqual(int[] val1, int[] val2) {
       return val1[0] == val2[0] && val1[1] == val2[1];
     }
   }

   private static class KeyHash2VirtualFileEnumerator extends PersistentBTreeEnumerator<int[]> {
     public KeyHash2VirtualFileEnumerator(File projectFile) throws IOException {
       super(projectFile, new IntPairInArrayKeyDescriptor(), 4096);
     }
   }
 }
	/*
	* Copyright 2000-2014 JetBrains s.r.o.
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	package com.intellij.util.indexing;

	import com.intellij.openapi.diagnostic.Logger;
	import com.intellij.openapi.progress.ProgressManager;
	import com.intellij.openapi.project.Project;
	import com.intellij.openapi.util.io.FileUtil;
	import com.intellij.psi.search.GlobalSearchScope;
	import com.intellij.psi.search.ProjectAndLibrariesScope;
	import com.intellij.psi.search.ProjectScopeImpl;
	import com.intellij.util.CommonProcessors;
	import com.intellij.util.Processor;
	import com.intellij.util.SystemProperties;
	import com.intellij.util.containers.SLRUCache;
	import com.intellij.util.io.*;
	import com.intellij.util.io.DataOutputStream;
	import gnu.trove.TIntHashSet;
	import gnu.trove.TIntProcedure;
	import org.jetbrains.annotations.NotNull;
	import org.jetbrains.annotations.Nullable;

	import java.io.*;
	import java.util.ArrayList;
	import java.util.Collection;
	import java.util.List;
	import java.util.concurrent.locks.Lock;
	import java.util.concurrent.locks.ReentrantLock;

	/**
	* @author Eugene Zhuravlev
	* Date: Dec 20, 2007
	*/
	public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Value>{
	private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapIndexStorage");
	private static final boolean ENABLE_CACHED_HASH_IDS = SystemProperties.getBooleanProperty("idea.index.no.cashed.hashids", true);
	private final boolean myBuildKeyHashToVirtualFileMapping;
	private PersistentMap<Key, ValueContainer<Value>> myMap;
	private PersistentBTreeEnumerator<int[]> myKeyHashToVirtualFileMapping;
	private SLRUCache<Key, ChangeTrackingValueContainer<Value>> myCache;
	private volatile int myLastScannedId;
	private final File myStorageFile;
	private final KeyDescriptor<Key> myKeyDescriptor;
	private final int myCacheSize;

	private final Lock l = new ReentrantLock();
	private final DataExternalizer<Value> myDataExternalizer;
	private final boolean myKeyIsUniqueForIndexedFile;

	public MapIndexStorage(@NotNull File storageFile,
	@NotNull KeyDescriptor<Key> keyDescriptor,
	@NotNull DataExternalizer<Value> valueExternalizer,
	final int cacheSize
	) throws IOException {
	this(storageFile, keyDescriptor, valueExternalizer, cacheSize, false, false);
	}

	public MapIndexStorage(@NotNull File storageFile,
	@NotNull KeyDescriptor<Key> keyDescriptor,
	@NotNull DataExternalizer<Value> valueExternalizer,
	final int cacheSize,
	boolean keyIsUniqueForIndexedFile,
	boolean buildKeyHashToVirtualFileMapping) throws IOException {
	myStorageFile = storageFile;
	myKeyDescriptor = keyDescriptor;
	myCacheSize = cacheSize;
	myDataExternalizer = valueExternalizer;
	myKeyIsUniqueForIndexedFile = keyIsUniqueForIndexedFile;
	myBuildKeyHashToVirtualFileMapping = buildKeyHashToVirtualFileMapping && FileBasedIndex.ourEnableTracingOfKeyHashToVirtualFileMapping;
	initMapAndCache();
	}

	private void initMapAndCache() throws IOException {
	final ValueContainerMap<Key, Value> map = new ValueContainerMap<Key, Value>(myStorageFile, myKeyDescriptor, myDataExternalizer,
	myKeyIsUniqueForIndexedFile);
	myCache = new SLRUCache<Key, ChangeTrackingValueContainer<Value>>(myCacheSize, (int)(Math.ceil(myCacheSize * 0.25)) /* 25% from the main cache size*/) {
	@Override
	@NotNull
	public ChangeTrackingValueContainer<Value> createValue(final Key key) {
	return new ChangeTrackingValueContainer<Value>(new ChangeTrackingValueContainer.Initializer<Value>() {
	@NotNull
	@Override
	public Object getLock() {
	return map.getDataAccessLock();
	}

	@Nullable
	@Override
	public ValueContainer<Value> compute() {
	ValueContainer<Value> value;
	try {
	value = map.get(key);
	if (value == null) {
	value = new ValueContainerImpl<Value>();
	}
	}
	catch (IOException e) {
	throw new RuntimeException(e);
	}
	return value;
	}
	});
	}

	@Override
	protected void onDropFromCache(final Key key, @NotNull final ChangeTrackingValueContainer<Value> valueContainer) {
	if (valueContainer.isDirty()) {
	try {
	map.put(key, valueContainer);
	}
	catch (IOException e) {
	throw new RuntimeException(e);
	}
	}
	}
	};

	myMap = map;

	myKeyHashToVirtualFileMapping = myBuildKeyHashToVirtualFileMapping ? new KeyHash2VirtualFileEnumerator(getProjectFile()) : null;
	}

	@NotNull
	private File getProjectFile() {
	return new File(myStorageFile.getPath() + ".project");
	}

	@Override
	public void flush() {
	l.lock();
	try {
	if (!myMap.isClosed()) {
	myCache.clear();
	if (myMap.isDirty()) myMap.force();
	}
	if (myKeyHashToVirtualFileMapping != null && myKeyHashToVirtualFileMapping.isDirty()) myKeyHashToVirtualFileMapping.force();
	}
	finally {
	l.unlock();
	}
	}

	@Override
	public void close() throws StorageException {
	try {
	flush();
	if (myKeyHashToVirtualFileMapping != null) myKeyHashToVirtualFileMapping.close();
	myMap.close();
	}
	catch (IOException e) {
	throw new StorageException(e);
	}
	catch (RuntimeException e) {
	final Throwable cause = e.getCause();
	if (cause instanceof IOException) {
	throw new StorageException(cause);
	}
	if (cause instanceof StorageException) {
	throw (StorageException)cause;
	}
	throw e;
	}
	}

	@Override
	public void clear() throws StorageException{
	try {
	myMap.close();
	if (myKeyHashToVirtualFileMapping != null) myKeyHashToVirtualFileMapping.close();
	}
	catch (IOException e) {
	LOG.error(e);
	}
	try {
	FileUtil.delete(myStorageFile);
	if (myKeyHashToVirtualFileMapping != null) IOUtil.deleteAllFilesStartingWith(getProjectFile());
	initMapAndCache();
	}
	catch (IOException e) {
	throw new StorageException(e);
	}
	catch (RuntimeException e) {
	final Throwable cause = e.getCause();
	if (cause instanceof IOException) {
	throw new StorageException(cause);
	}
	if (cause instanceof StorageException) {
	throw (StorageException)cause;
	}
	throw e;
	}
	}

	@Override
	public boolean processKeys(@NotNull final Processor<Key> processor, GlobalSearchScope scope, final IdFilter idFilter) throws StorageException {
	l.lock();
	try {
	myCache.clear(); // this will ensure that all new keys are made into the map
	if (myBuildKeyHashToVirtualFileMapping && idFilter != null) {
	TIntHashSet hashMaskSet = null;
	long l = System.currentTimeMillis();

	File fileWithCaches = getSavedProjectFileValueIds(myLastScannedId, scope);
	final boolean useCachedHashIds = ENABLE_CACHED_HASH_IDS &&
	(scope instanceof ProjectScopeImpl \|\| scope instanceof ProjectAndLibrariesScope) &&
	fileWithCaches != null;
	int id = myKeyHashToVirtualFileMapping.getLargestId();

	if (useCachedHashIds && id == myLastScannedId) {
	try {
	hashMaskSet = loadHashedIds(fileWithCaches);
	} catch (IOException ignored) {
	}
	}

	if (hashMaskSet == null) {
	if (useCachedHashIds && myLastScannedId != 0) {
	FileUtil.asyncDelete(fileWithCaches);
	}

	hashMaskSet = new TIntHashSet(1000);
	final TIntHashSet finalHashMaskSet = hashMaskSet;
	myKeyHashToVirtualFileMapping.iterateData(new Processor<int[]>() {
	@Override
	public boolean process(int[] key) {
	if (!idFilter.containsFileId(key[1])) return true;
	finalHashMaskSet.add(key[0]);
	ProgressManager.checkCanceled();
	return true;
	}
	});

	if (useCachedHashIds) {
	saveHashedIds(hashMaskSet, id, scope);
	}
	}

	if (LOG.isDebugEnabled()) {
	LOG.debug("Scanned keyHashToVirtualFileMapping of " + myStorageFile + " for " + (System.currentTimeMillis() - l));
	}
	final TIntHashSet finalHashMaskSet = hashMaskSet;
	return myMap.processKeys(new Processor<Key>() {
	@Override
	public boolean process(Key key) {
	if (!finalHashMaskSet.contains(myKeyDescriptor.getHashCode(key))) return true;
	return processor.process(key);
	}
	});
	}
	return myMap.processKeys(processor);
	}
	catch (IOException e) {
	throw new StorageException(e);
	}
	catch (RuntimeException e) {
	final Throwable cause = e.getCause();
	if (cause instanceof IOException) {
	throw new StorageException(cause);
	}
	if (cause instanceof StorageException) {
	throw (StorageException)cause;
	}
	throw e;
	}
	finally {
	l.unlock();
	}
	}

	@NotNull
	private static TIntHashSet loadHashedIds(@NotNull File fileWithCaches) throws IOException {
	DataInputStream inputStream = null;
	try {
	inputStream = new DataInputStream(new BufferedInputStream(new FileInputStream(fileWithCaches)));
	int capacity = DataInputOutputUtil.readINT(inputStream);
	TIntHashSet hashMaskSet = new TIntHashSet(capacity);
	while(capacity > 0) {
	hashMaskSet.add(DataInputOutputUtil.readINT(inputStream));
	--capacity;
	}
	inputStream.close();
	return hashMaskSet;
	}
	finally {
	if (inputStream != null) {
	try {
	inputStream.close();
	}
	catch (IOException ignored) {}
	}
	}
	}

	private void saveHashedIds(@NotNull TIntHashSet hashMaskSet, int largestId, @NotNull GlobalSearchScope scope) {
	File newFileWithCaches = getSavedProjectFileValueIds(largestId, scope);
	assert newFileWithCaches != null;
	DataOutputStream stream = null;

	boolean savedSuccessfully = false;
	try {
	stream = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(newFileWithCaches)));
	DataInputOutputUtil.writeINT(stream, hashMaskSet.size());
	final DataOutputStream finalStream = stream;
	savedSuccessfully = hashMaskSet.forEach(new TIntProcedure() {
	@Override
	public boolean execute(int value) {
	try {
	DataInputOutputUtil.writeINT(finalStream, value);
	return true;
	} catch (IOException ex) {
	return false;
	}
	}
	});
	}
	catch (IOException ignored) {
	}
	finally {
	if (stream != null) {
	try {
	stream.close();
	if (savedSuccessfully) myLastScannedId = largestId;
	}
	catch (IOException ignored) {}
	}
	}
	}

	@Nullable
	private File getSavedProjectFileValueIds(int id, @NotNull GlobalSearchScope scope) {
	Project project = scope.getProject();
	if (project == null) return null;
	return new File(myStorageFile.getPath() + ".project."+project.hashCode() + "."+id + "." + scope.isSearchInLibraries());
	}

	@NotNull
	@Override
	public Collection<Key> getKeys() throws StorageException {
	List<Key> keys = new ArrayList<Key>();
	processKeys(new CommonProcessors.CollectProcessor<Key>(keys), null, null);
	return keys;
	}

	@Override
	@NotNull
	public ChangeTrackingValueContainer<Value> read(final Key key) throws StorageException {
	l.lock();
	try {
	return myCache.get(key);
	}
	catch (RuntimeException e) {
	final Throwable cause = e.getCause();
	if (cause instanceof IOException) {
	throw new StorageException(cause);
	}
	if (cause instanceof StorageException) {
	throw (StorageException)cause;
	}
	throw e;
	}
	finally {
	l.unlock();
	}
	}

	@Override
	public void addValue(final Key key, final int inputId, final Value value) throws StorageException {
	try {
	if (myKeyHashToVirtualFileMapping != null) {
	myKeyHashToVirtualFileMapping.enumerate(new int[] { myKeyDescriptor.getHashCode(key), inputId });
	}

	myMap.markDirty();
	if (!myKeyIsUniqueForIndexedFile) {
	read(key).addValue(inputId, value);
	return;
	}

	ChangeTrackingValueContainer<Value> cached;
	try {
	l.lock();
	cached = myCache.getIfCached(key);
	} finally {
	l.unlock();
	}

	if (cached != null) {
	cached.addValue(inputId, value);
	return;
	}
	// do not pollute the cache with keys unique to indexed file
	ChangeTrackingValueContainer<Value> valueContainer = new ChangeTrackingValueContainer<Value>(null);
	valueContainer.addValue(inputId, value);
	myMap.put(key, valueContainer);
	}
	catch (IOException e) {
	throw new StorageException(e);
	}
	}

	@Override
	public void removeAllValues(@NotNull Key key, int inputId) throws StorageException {
	try {
	myMap.markDirty();
	// important: assuming the key exists in the index
	read(key).removeAssociatedValue(inputId);
	}
	catch (IOException e) {
	throw new StorageException(e);
	}
	}

	private static class IntPairInArrayKeyDescriptor implements KeyDescriptor<int[]>, DifferentSerializableBytesImplyNonEqualityPolicy {
	@Override
	public void save(@NotNull DataOutput out, int[] value) throws IOException {
	DataInputOutputUtil.writeINT(out, value[0]);
	DataInputOutputUtil.writeINT(out, value[1]);
	}

	@Override
	public int[] read(@NotNull DataInput in) throws IOException {
	return new int[] {DataInputOutputUtil.readINT(in), DataInputOutputUtil.readINT(in)};
	}

	@Override
	public int getHashCode(int[] value) {
	return value[0] * 31 + value[1];
	}

	@Override
	public boolean isEqual(int[] val1, int[] val2) {
	return val1[0] == val2[0] && val1[1] == val2[1];
	}
	}

	private static class KeyHash2VirtualFileEnumerator extends PersistentBTreeEnumerator<int[]> {
	public KeyHash2VirtualFileEnumerator(File projectFile) throws IOException {
	super(projectFile, new IntPairInArrayKeyDescriptor(), 4096);
	}
	}
	}