platform/util/src/com/intellij/util/io/PersistentEnumeratorBase.java - platform/tools/idea - Git at Google

 /*
  * Copyright 2000-2013 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package com.intellij.util.io;

 import com.intellij.openapi.Forceable;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.util.io.FileUtil;
 import com.intellij.util.CommonProcessors;
 import com.intellij.util.IncorrectOperationException;
 import com.intellij.util.Processor;
 import com.intellij.util.containers.SLRUMap;
 import com.intellij.util.containers.ShareableKey;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jetbrains.annotations.TestOnly;

 import java.io.Closeable;
 import java.io.File;
 import java.io.Flushable;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;

 /**
  * @author max
  * @author jeka
  */
 @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
 public abstract class PersistentEnumeratorBase<Data> implements Forceable, Closeable {
   protected static final Logger LOG = Logger.getInstance("#com.intellij.util.io.PersistentEnumerator");
   protected static final int NULL_ID = 0;

   private static final int META_DATA_OFFSET = 4;
   protected static final int DATA_START = META_DATA_OFFSET + 16;

   protected final ResizeableMappedFile myStorage;
   private final boolean myAssumeDifferentSerializedBytesMeansObjectsInequality;
   private final AppendableStorageBackedByResizableMappedFile myKeyStorage;

   private boolean myClosed = false;
   private boolean myDirty = false;
   protected final KeyDescriptor<Data> myDataDescriptor;

   private static final CacheKey ourFlyweight = new FlyweightKey();

   protected final File myFile;
   private boolean myCorrupted = false;
   private final int myInitialSize;
   private final Version myVersion;
   private RecordBufferHandler<PersistentEnumeratorBase> myRecordHandler;
   private volatile boolean myDirtyStatusUpdateInProgress;
   private Flushable myMarkCleanCallback;
   private final boolean myDoCaching;

   public static class Version {
     private final int correctlyClosedMagic;
     private final int dirtyMagic;

     public Version(int _correctlyClosedMagic, int _dirtyMagic) {
       correctlyClosedMagic = _correctlyClosedMagic;
       dirtyMagic = _dirtyMagic;
       assert correctlyClosedMagic != dirtyMagic;
     }
   }

   public abstract static class RecordBufferHandler<T extends PersistentEnumeratorBase> {
     abstract int recordWriteOffset(T enumerator, byte[] buf);
     @NotNull
     abstract byte[] getRecordBuffer(T enumerator);
     abstract void setupRecord(T enumerator, int hashCode, final int dataOffset, final byte[] buf);
   }

   private static class CacheKey implements ShareableKey {
     public PersistentEnumeratorBase owner;
     public Object key;

     private CacheKey(Object key, PersistentEnumeratorBase owner) {
       this.key = key;
       this.owner = owner;
     }

     @Override
     public ShareableKey getStableCopy() {
       return this;
     }

     public boolean equals(final Object o) {
       if (this == o) return true;
       if (!(o instanceof CacheKey)) return false;

       final CacheKey cacheKey = (CacheKey)o;

       if (!key.equals(cacheKey.key)) return false;
       if (!owner.equals(cacheKey.owner)) return false;

       return true;
     }

     public int hashCode() {
       return key.hashCode();
     }
   }

   private static CacheKey sharedKey(Object key, PersistentEnumeratorBase owner) {
     ourFlyweight.key = key;
     ourFlyweight.owner = owner;
     return ourFlyweight;
   }

   private static final int ENUMERATION_CACHE_SIZE;
   static {
     String property = System.getProperty("idea.enumerationCacheSize");
     ENUMERATION_CACHE_SIZE = property == null ? 8192 : Integer.valueOf(property);
   }

   private static final SLRUMap<Object, Integer> ourEnumerationCache = new SLRUMap<Object, Integer>(ENUMERATION_CACHE_SIZE, ENUMERATION_CACHE_SIZE);

   @TestOnly
   public static void clearCacheForTests() {
     ourEnumerationCache.clear();
   }

   public static class CorruptedException extends IOException {
     @SuppressWarnings({"HardCodedStringLiteral"})
     public CorruptedException(File file) {
       this("PersistentEnumerator storage corrupted " + file.getPath());
     }

     protected CorruptedException(String message) {
       super(message);
     }
   }

   public static class VersionUpdatedException extends CorruptedException {
     @SuppressWarnings({"HardCodedStringLiteral"})
     public VersionUpdatedException(File file) {
       super("PersistentEnumerator storage corrupted " + file.getPath());
     }
   }

   public PersistentEnumeratorBase(@NotNull File file,
                                   @NotNull ResizeableMappedFile storage,
                                   @NotNull KeyDescriptor<Data> dataDescriptor,
                                   int initialSize,
                                   @NotNull Version version,
                                   @NotNull RecordBufferHandler<? extends PersistentEnumeratorBase> recordBufferHandler,
                                   boolean doCaching) throws IOException {
     myDataDescriptor = dataDescriptor;
     myFile = file;
     myInitialSize = initialSize;
     myVersion = version;
     myRecordHandler = (RecordBufferHandler<PersistentEnumeratorBase>)recordBufferHandler;
     myDoCaching = doCaching;

     if (!file.exists()) {
       FileUtil.delete(keystreamFile());
       if (!FileUtil.createIfDoesntExist(file)) {
         throw new IOException("Cannot create empty file: " + file);
       }
     }

     myStorage = storage;

     lockStorage();
     try {
       if (myStorage.length() == 0) {
         try {
           markDirty(true);
           putMetaData(0);
           putMetaData2(0);
           setupEmptyFile();
         }
         catch (RuntimeException e) {
           LOG.info(e);
           myStorage.close();
           if (e.getCause() instanceof IOException) {
             throw (IOException)e.getCause();
           }
           throw e;
         }
         catch (IOException e) {
           LOG.info(e);
           myStorage.close();
           throw e;
         }
         catch (Exception e) {
           LOG.info(e);
           myStorage.close();
           throw new CorruptedException(file);
         }
       }
       else {
         int sign;
         try {
           sign = myStorage.getInt(0);
         }
         catch(Exception e) {
           LOG.info(e);
           sign = myVersion.dirtyMagic;
         }
         if (sign != myVersion.correctlyClosedMagic) {
           myStorage.close();
           if (sign != myVersion.dirtyMagic) throw new VersionUpdatedException(file);
           throw new CorruptedException(file);
         }
       }
     }
     finally {
       unlockStorage();
     }

     if (myDataDescriptor instanceof InlineKeyDescriptor) {
       myKeyStorage = null;
     }
     else {
       try {
         myKeyStorage = new AppendableStorageBackedByResizableMappedFile(keystreamFile(), initialSize, myStorage.getPagedFileStorage().getStorageLockContext(), PagedFileStorage.MB, false);
       }
       catch (IOException e) {
         myStorage.close();
         throw e;
       }
       catch (Throwable e) {
         LOG.info(e);
         myStorage.close();
         throw new CorruptedException(file);
       }
     }
     myAssumeDifferentSerializedBytesMeansObjectsInequality = myDataDescriptor instanceof DifferentSerializableBytesImplyNonEqualityPolicy;
   }

   public void lockStorage() {
     myStorage.getPagedFileStorage().lock();
   }

   public void unlockStorage() {
     myStorage.getPagedFileStorage().unlock();
   }

   protected abstract void setupEmptyFile() throws IOException;

   @NotNull
   public final RecordBufferHandler<PersistentEnumeratorBase> getRecordHandler() {
     return myRecordHandler;
   }

   public void setRecordHandler(@NotNull RecordBufferHandler<PersistentEnumeratorBase> recordHandler) {
     myRecordHandler = recordHandler;
   }

   public void setMarkCleanCallback(Flushable markCleanCallback) {
     myMarkCleanCallback = markCleanCallback;
   }

   public Data getValue(int keyId, int processingKey) throws IOException {
     return valueOf(keyId);
   }

   protected int tryEnumerate(Data value) throws IOException {
     return doEnumerate(value, true, false);
   }

   private int doEnumerate(Data value, boolean onlyCheckForExisting, boolean saveNewValue) throws IOException {
     if (myDoCaching && !saveNewValue) {
       synchronized (ourEnumerationCache) {
         final Integer cachedId = ourEnumerationCache.get(sharedKey(value, this));
         if (cachedId != null) return cachedId.intValue();
       }
     }

     final int id;
     try {
       id = enumerateImpl(value, onlyCheckForExisting, saveNewValue);
     }
     catch (IOException io) {
       markCorrupted();
       throw io;
     }
     catch (Throwable e) {
       markCorrupted();
       LOG.info(e);
       throw new IOException(e);
     }

     if (myDoCaching && id != NULL_ID) {
       synchronized (ourEnumerationCache) {
         ourEnumerationCache.put(new CacheKey(value, this), id);
       }
     }

     return id;
   }

   public int enumerate(Data value) throws IOException {
     return doEnumerate(value, false, false);
   }

   public interface DataFilter {
     boolean accept(int id);
   }

   protected void putMetaData(long data) throws IOException {
     lockStorage();
     try {
       if (myStorage.length() < META_DATA_OFFSET + 8 || getMetaData() != data) myStorage.putLong(META_DATA_OFFSET, data);
     }
     finally {
       unlockStorage();
     }
   }

   protected long getMetaData() throws IOException {
     lockStorage();
     try {
       return myStorage.getLong(META_DATA_OFFSET);
     }
     finally {
       unlockStorage();
     }
   }

   protected void putMetaData2(long data) throws IOException {
     lockStorage();
     try {
       if (myStorage.length() < META_DATA_OFFSET + 16 || getMetaData2() != data) myStorage.putLong(META_DATA_OFFSET + 8, data);
     }
     finally {
       unlockStorage();
     }
   }

   protected long getMetaData2() throws IOException {
     lockStorage();
     try {
       return myStorage.getLong(META_DATA_OFFSET + 8);
     }
     finally {
       unlockStorage();
     }
   }

   public boolean processAllDataObject(final Processor<Data> processor, @Nullable final DataFilter filter) throws IOException {
     return traverseAllRecords(new RecordsProcessor() {
       @Override
       public boolean process(final int record) throws IOException {
         if (filter == null || filter.accept(record)) {
           return processor.process(valueOf(record));
         }
         return true;
       }
     });

   }

   public Collection<Data> getAllDataObjects(@Nullable final DataFilter filter) throws IOException {
     final List<Data> values = new ArrayList<Data>();
     processAllDataObject(new CommonProcessors.CollectProcessor<Data>(values), filter);
     return values;
   }

   public abstract static class RecordsProcessor {
     private int myKey;

     public abstract boolean process(int record) throws IOException;
     void setCurrentKey(int key) {
       myKey = key;
     }
     int getCurrentKey() {
       return myKey;
     }
   }

   public abstract boolean traverseAllRecords(RecordsProcessor p) throws IOException;

   protected abstract int enumerateImpl(final Data value, final boolean onlyCheckForExisting, boolean saveNewValue) throws IOException;

   protected boolean isKeyAtIndex(final Data value, final int idx) throws IOException {
     if (myKeyStorage == null) return false;

     // check if previous serialized state is the same as for value
     // this is much faster than myDataDescriptor.isEqualTo(valueOf(idx), value) for identical objects
     // TODO: key storage lock
     final int addr = indexToAddr(idx);

     if (myKeyStorage.checkBytesAreTheSame(addr, value, myDataDescriptor)) return true;
     if (myAssumeDifferentSerializedBytesMeansObjectsInequality) return false;
     return myDataDescriptor.isEqual(valueOf(idx), value);
   }

   protected int writeData(final Data value, int hashCode) {
     try {
       markDirty(true);

       final int dataOff = doWriteData(value);

       return setupValueId(hashCode, dataOff);
     }
     catch (IOException e) {
       throw new RuntimeException(e);
     }
   }

   public int getLargestId() {
     assert myKeyStorage != null;
     return myKeyStorage.getCurrentLength();
   }

   protected int doWriteData(Data value) throws IOException {
     if (myKeyStorage != null) {
       return myKeyStorage.append(value, myDataDescriptor);

     }
     return ((InlineKeyDescriptor<Data>)myDataDescriptor).toInt(value);
   }

   protected int setupValueId(int hashCode, int dataOff) {
     final byte[] buf = myRecordHandler.getRecordBuffer(this);
     myRecordHandler.setupRecord(this, hashCode, dataOff, buf);
     final int pos = myRecordHandler.recordWriteOffset(this, buf);
     myStorage.put(pos, buf, 0, buf.length);

     return pos;
   }

   public boolean iterateData(final Processor<Data> processor) throws IOException {
     if (myKeyStorage == null) {
       throw new UnsupportedOperationException("Iteration over InlineIntegerKeyDescriptors is not supported");
     }

     lockStorage(); // todo locking in key storage
     try {
       myKeyStorage.force();
     }
     finally {
       unlockStorage();
     }

     return myKeyStorage.processAll(processor, myDataDescriptor);
   }

   private File keystreamFile() {
     return new File(myFile.getPath() + ".keystream");
   }

   public Data valueOf(int idx) throws IOException {
     lockStorage();
     try {
       int addr = indexToAddr(idx);

       if (myKeyStorage == null) return ((InlineKeyDescriptor<Data>)myDataDescriptor).fromInt(addr);
       return myKeyStorage.read(addr, myDataDescriptor);
     }
     catch (IOException io) {
       markCorrupted();
       throw io;
     }
     catch (Throwable e) {
       markCorrupted();
       throw new RuntimeException(e);
     }
     finally {
       unlockStorage();
     }
   }

   int reenumerate(Data key) throws IOException {
     if (!canReEnumerate()) throw new IncorrectOperationException();
     return doEnumerate(key, false, true);
   }

   boolean canReEnumerate() {
     return false;
   }

   protected abstract int indexToAddr(int idx);

   @Override
   public synchronized void close() throws IOException {
     lockStorage();
     try {
       if (!myClosed) {
         myClosed = true;
         doClose();
       }
     }
     finally {
       unlockStorage();
     }
   }

   protected void doClose() throws IOException {
     try {
       if (myKeyStorage != null) {
         myKeyStorage.close();
       }
       flush();
     }
     finally {
       myStorage.close();
     }
   }

   public synchronized boolean isClosed() {
     return myClosed;
   }

   @Override
   public synchronized boolean isDirty() {
     return myDirty;
   }

   private synchronized void flush() throws IOException {
     lockStorage();
     try {
       if (myStorage.isDirty() || isDirty()) {
         doFlush();
       }
     }
     finally {
       unlockStorage();
     }
   }

   protected void doFlush() throws IOException {
     markDirty(false);
     myStorage.force();
   }

   @Override
   public synchronized void force() {
     lockStorage();

     try {
       if (myKeyStorage != null) {
         myKeyStorage.force();
       }
       flush();
     }
     catch (IOException e) {
       throw new RuntimeException(e);
     }
     finally {
       unlockStorage();
     }
   }

   protected final void markDirty(boolean dirty) throws IOException {
     //assert Thread.holdsLock(this) || Thread.holdsLock(ourLock); // we hold one lock or another so can access myDirty
     if (dirty && myDirty && !myDirtyStatusUpdateInProgress) return;
     lockStorage();
     try {
       if (myDirty) {
         if (!dirty) {
           myDirtyStatusUpdateInProgress = true;
           if (myMarkCleanCallback != null) myMarkCleanCallback.flush();
           if (!myCorrupted) {
             myStorage.putInt(0, myVersion.correctlyClosedMagic);
             myDirty = false;
           }
           myDirtyStatusUpdateInProgress = false;
         }
       }
       else {
         if (dirty) {
           myDirtyStatusUpdateInProgress = true;
           myStorage.putInt(0, myVersion.dirtyMagic);
           myDirtyStatusUpdateInProgress = false;
           myDirty = true;
         }
       }
     }
     finally {
       unlockStorage();
     }
   }

   protected synchronized void markCorrupted() {
     if (!myCorrupted) {
       myCorrupted = true;
       try {
         markDirty(true);
         force();
       }
       catch (IOException e) {
         // ignore...
       }
     }
   }

   private static class FlyweightKey extends CacheKey {
     public FlyweightKey() {
       super(null, null);
     }

     @Override
     public ShareableKey getStableCopy() {
       return new CacheKey(key, owner);
     }
   }
 }
	/*
	* Copyright 2000-2013 JetBrains s.r.o.
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package com.intellij.util.io;

	import com.intellij.openapi.Forceable;
	import com.intellij.openapi.diagnostic.Logger;
	import com.intellij.openapi.util.io.FileUtil;
	import com.intellij.util.CommonProcessors;
	import com.intellij.util.IncorrectOperationException;
	import com.intellij.util.Processor;
	import com.intellij.util.containers.SLRUMap;
	import com.intellij.util.containers.ShareableKey;
	import org.jetbrains.annotations.NotNull;
	import org.jetbrains.annotations.Nullable;
	import org.jetbrains.annotations.TestOnly;

	import java.io.Closeable;
	import java.io.File;
	import java.io.Flushable;
	import java.io.IOException;
	import java.util.ArrayList;
	import java.util.Collection;
	import java.util.List;

	/**
	* @author max
	* @author jeka
	*/
	@SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
	public abstract class PersistentEnumeratorBase<Data> implements Forceable, Closeable {
	protected static final Logger LOG = Logger.getInstance("#com.intellij.util.io.PersistentEnumerator");
	protected static final int NULL_ID = 0;

	private static final int META_DATA_OFFSET = 4;
	protected static final int DATA_START = META_DATA_OFFSET + 16;

	protected final ResizeableMappedFile myStorage;
	private final boolean myAssumeDifferentSerializedBytesMeansObjectsInequality;
	private final AppendableStorageBackedByResizableMappedFile myKeyStorage;

	private boolean myClosed = false;
	private boolean myDirty = false;
	protected final KeyDescriptor<Data> myDataDescriptor;

	private static final CacheKey ourFlyweight = new FlyweightKey();

	protected final File myFile;
	private boolean myCorrupted = false;
	private final int myInitialSize;
	private final Version myVersion;
	private RecordBufferHandler<PersistentEnumeratorBase> myRecordHandler;
	private volatile boolean myDirtyStatusUpdateInProgress;
	private Flushable myMarkCleanCallback;
	private final boolean myDoCaching;

	public static class Version {
	private final int correctlyClosedMagic;
	private final int dirtyMagic;

	public Version(int _correctlyClosedMagic, int _dirtyMagic) {
	correctlyClosedMagic = _correctlyClosedMagic;
	dirtyMagic = _dirtyMagic;
	assert correctlyClosedMagic != dirtyMagic;
	}
	}

	public abstract static class RecordBufferHandler<T extends PersistentEnumeratorBase> {
	abstract int recordWriteOffset(T enumerator, byte[] buf);
	@NotNull
	abstract byte[] getRecordBuffer(T enumerator);
	abstract void setupRecord(T enumerator, int hashCode, final int dataOffset, final byte[] buf);
	}

	private static class CacheKey implements ShareableKey {
	public PersistentEnumeratorBase owner;
	public Object key;

	private CacheKey(Object key, PersistentEnumeratorBase owner) {
	this.key = key;
	this.owner = owner;
	}

	@Override
	public ShareableKey getStableCopy() {
	return this;
	}

	public boolean equals(final Object o) {
	if (this == o) return true;
	if (!(o instanceof CacheKey)) return false;

	final CacheKey cacheKey = (CacheKey)o;

	if (!key.equals(cacheKey.key)) return false;
	if (!owner.equals(cacheKey.owner)) return false;

	return true;
	}

	public int hashCode() {
	return key.hashCode();
	}
	}

	private static CacheKey sharedKey(Object key, PersistentEnumeratorBase owner) {
	ourFlyweight.key = key;
	ourFlyweight.owner = owner;
	return ourFlyweight;
	}

	private static final int ENUMERATION_CACHE_SIZE;
	static {
	String property = System.getProperty("idea.enumerationCacheSize");
	ENUMERATION_CACHE_SIZE = property == null ? 8192 : Integer.valueOf(property);
	}

	private static final SLRUMap<Object, Integer> ourEnumerationCache = new SLRUMap<Object, Integer>(ENUMERATION_CACHE_SIZE, ENUMERATION_CACHE_SIZE);

	@TestOnly
	public static void clearCacheForTests() {
	ourEnumerationCache.clear();
	}

	public static class CorruptedException extends IOException {
	@SuppressWarnings({"HardCodedStringLiteral"})
	public CorruptedException(File file) {
	this("PersistentEnumerator storage corrupted " + file.getPath());
	}

	protected CorruptedException(String message) {
	super(message);
	}
	}

	public static class VersionUpdatedException extends CorruptedException {
	@SuppressWarnings({"HardCodedStringLiteral"})
	public VersionUpdatedException(File file) {
	super("PersistentEnumerator storage corrupted " + file.getPath());
	}
	}

	public PersistentEnumeratorBase(@NotNull File file,
	@NotNull ResizeableMappedFile storage,
	@NotNull KeyDescriptor<Data> dataDescriptor,
	int initialSize,
	@NotNull Version version,
	@NotNull RecordBufferHandler<? extends PersistentEnumeratorBase> recordBufferHandler,
	boolean doCaching) throws IOException {
	myDataDescriptor = dataDescriptor;
	myFile = file;
	myInitialSize = initialSize;
	myVersion = version;
	myRecordHandler = (RecordBufferHandler<PersistentEnumeratorBase>)recordBufferHandler;
	myDoCaching = doCaching;

	if (!file.exists()) {
	FileUtil.delete(keystreamFile());
	if (!FileUtil.createIfDoesntExist(file)) {
	throw new IOException("Cannot create empty file: " + file);
	}
	}

	myStorage = storage;

	lockStorage();
	try {
	if (myStorage.length() == 0) {
	try {
	markDirty(true);
	putMetaData(0);
	putMetaData2(0);
	setupEmptyFile();
	}
	catch (RuntimeException e) {
	LOG.info(e);
	myStorage.close();
	if (e.getCause() instanceof IOException) {
	throw (IOException)e.getCause();
	}
	throw e;
	}
	catch (IOException e) {
	LOG.info(e);
	myStorage.close();
	throw e;
	}
	catch (Exception e) {
	LOG.info(e);
	myStorage.close();
	throw new CorruptedException(file);
	}
	}
	else {
	int sign;
	try {
	sign = myStorage.getInt(0);
	}
	catch(Exception e) {
	LOG.info(e);
	sign = myVersion.dirtyMagic;
	}
	if (sign != myVersion.correctlyClosedMagic) {
	myStorage.close();
	if (sign != myVersion.dirtyMagic) throw new VersionUpdatedException(file);
	throw new CorruptedException(file);
	}
	}
	}
	finally {
	unlockStorage();
	}

	if (myDataDescriptor instanceof InlineKeyDescriptor) {
	myKeyStorage = null;
	}
	else {
	try {
	myKeyStorage = new AppendableStorageBackedByResizableMappedFile(keystreamFile(), initialSize, myStorage.getPagedFileStorage().getStorageLockContext(), PagedFileStorage.MB, false);
	}
	catch (IOException e) {
	myStorage.close();
	throw e;
	}
	catch (Throwable e) {
	LOG.info(e);
	myStorage.close();
	throw new CorruptedException(file);
	}
	}
	myAssumeDifferentSerializedBytesMeansObjectsInequality = myDataDescriptor instanceof DifferentSerializableBytesImplyNonEqualityPolicy;
	}

	public void lockStorage() {
	myStorage.getPagedFileStorage().lock();
	}

	public void unlockStorage() {
	myStorage.getPagedFileStorage().unlock();
	}

	protected abstract void setupEmptyFile() throws IOException;

	@NotNull
	public final RecordBufferHandler<PersistentEnumeratorBase> getRecordHandler() {
	return myRecordHandler;
	}

	public void setRecordHandler(@NotNull RecordBufferHandler<PersistentEnumeratorBase> recordHandler) {
	myRecordHandler = recordHandler;
	}

	public void setMarkCleanCallback(Flushable markCleanCallback) {
	myMarkCleanCallback = markCleanCallback;
	}

	public Data getValue(int keyId, int processingKey) throws IOException {
	return valueOf(keyId);
	}

	protected int tryEnumerate(Data value) throws IOException {
	return doEnumerate(value, true, false);
	}

	private int doEnumerate(Data value, boolean onlyCheckForExisting, boolean saveNewValue) throws IOException {
	if (myDoCaching && !saveNewValue) {
	synchronized (ourEnumerationCache) {
	final Integer cachedId = ourEnumerationCache.get(sharedKey(value, this));
	if (cachedId != null) return cachedId.intValue();
	}
	}

	final int id;
	try {
	id = enumerateImpl(value, onlyCheckForExisting, saveNewValue);
	}
	catch (IOException io) {
	markCorrupted();
	throw io;
	}
	catch (Throwable e) {
	markCorrupted();
	LOG.info(e);
	throw new IOException(e);
	}

	if (myDoCaching && id != NULL_ID) {
	synchronized (ourEnumerationCache) {
	ourEnumerationCache.put(new CacheKey(value, this), id);
	}
	}

	return id;
	}

	public int enumerate(Data value) throws IOException {
	return doEnumerate(value, false, false);
	}

	public interface DataFilter {
	boolean accept(int id);
	}

	protected void putMetaData(long data) throws IOException {
	lockStorage();
	try {
	if (myStorage.length() < META_DATA_OFFSET + 8 \|\| getMetaData() != data) myStorage.putLong(META_DATA_OFFSET, data);
	}
	finally {
	unlockStorage();
	}
	}

	protected long getMetaData() throws IOException {
	lockStorage();
	try {
	return myStorage.getLong(META_DATA_OFFSET);
	}
	finally {
	unlockStorage();
	}
	}

	protected void putMetaData2(long data) throws IOException {
	lockStorage();
	try {
	if (myStorage.length() < META_DATA_OFFSET + 16 \|\| getMetaData2() != data) myStorage.putLong(META_DATA_OFFSET + 8, data);
	}
	finally {
	unlockStorage();
	}
	}

	protected long getMetaData2() throws IOException {
	lockStorage();
	try {
	return myStorage.getLong(META_DATA_OFFSET + 8);
	}
	finally {
	unlockStorage();
	}
	}

	public boolean processAllDataObject(final Processor<Data> processor, @Nullable final DataFilter filter) throws IOException {
	return traverseAllRecords(new RecordsProcessor() {
	@Override
	public boolean process(final int record) throws IOException {
	if (filter == null \|\| filter.accept(record)) {
	return processor.process(valueOf(record));
	}
	return true;
	}
	});

	}

	public Collection<Data> getAllDataObjects(@Nullable final DataFilter filter) throws IOException {
	final List<Data> values = new ArrayList<Data>();
	processAllDataObject(new CommonProcessors.CollectProcessor<Data>(values), filter);
	return values;
	}

	public abstract static class RecordsProcessor {
	private int myKey;

	public abstract boolean process(int record) throws IOException;
	void setCurrentKey(int key) {
	myKey = key;
	}
	int getCurrentKey() {
	return myKey;
	}
	}

	public abstract boolean traverseAllRecords(RecordsProcessor p) throws IOException;

	protected abstract int enumerateImpl(final Data value, final boolean onlyCheckForExisting, boolean saveNewValue) throws IOException;

	protected boolean isKeyAtIndex(final Data value, final int idx) throws IOException {
	if (myKeyStorage == null) return false;

	// check if previous serialized state is the same as for value
	// this is much faster than myDataDescriptor.isEqualTo(valueOf(idx), value) for identical objects
	// TODO: key storage lock
	final int addr = indexToAddr(idx);

	if (myKeyStorage.checkBytesAreTheSame(addr, value, myDataDescriptor)) return true;
	if (myAssumeDifferentSerializedBytesMeansObjectsInequality) return false;
	return myDataDescriptor.isEqual(valueOf(idx), value);
	}

	protected int writeData(final Data value, int hashCode) {
	try {
	markDirty(true);

	final int dataOff = doWriteData(value);

	return setupValueId(hashCode, dataOff);
	}
	catch (IOException e) {
	throw new RuntimeException(e);
	}
	}

	public int getLargestId() {
	assert myKeyStorage != null;
	return myKeyStorage.getCurrentLength();
	}

	protected int doWriteData(Data value) throws IOException {
	if (myKeyStorage != null) {
	return myKeyStorage.append(value, myDataDescriptor);

	}
	return ((InlineKeyDescriptor<Data>)myDataDescriptor).toInt(value);
	}

	protected int setupValueId(int hashCode, int dataOff) {
	final byte[] buf = myRecordHandler.getRecordBuffer(this);
	myRecordHandler.setupRecord(this, hashCode, dataOff, buf);
	final int pos = myRecordHandler.recordWriteOffset(this, buf);
	myStorage.put(pos, buf, 0, buf.length);

	return pos;
	}

	public boolean iterateData(final Processor<Data> processor) throws IOException {
	if (myKeyStorage == null) {
	throw new UnsupportedOperationException("Iteration over InlineIntegerKeyDescriptors is not supported");
	}

	lockStorage(); // todo locking in key storage
	try {
	myKeyStorage.force();
	}
	finally {
	unlockStorage();
	}

	return myKeyStorage.processAll(processor, myDataDescriptor);
	}

	private File keystreamFile() {
	return new File(myFile.getPath() + ".keystream");
	}

	public Data valueOf(int idx) throws IOException {
	lockStorage();
	try {
	int addr = indexToAddr(idx);

	if (myKeyStorage == null) return ((InlineKeyDescriptor<Data>)myDataDescriptor).fromInt(addr);
	return myKeyStorage.read(addr, myDataDescriptor);
	}
	catch (IOException io) {
	markCorrupted();
	throw io;
	}
	catch (Throwable e) {
	markCorrupted();
	throw new RuntimeException(e);
	}
	finally {
	unlockStorage();
	}
	}

	int reenumerate(Data key) throws IOException {
	if (!canReEnumerate()) throw new IncorrectOperationException();
	return doEnumerate(key, false, true);
	}

	boolean canReEnumerate() {
	return false;
	}

	protected abstract int indexToAddr(int idx);

	@Override
	public synchronized void close() throws IOException {
	lockStorage();
	try {
	if (!myClosed) {
	myClosed = true;
	doClose();
	}
	}
	finally {
	unlockStorage();
	}
	}

	protected void doClose() throws IOException {
	try {
	if (myKeyStorage != null) {
	myKeyStorage.close();
	}
	flush();
	}
	finally {
	myStorage.close();
	}
	}

	public synchronized boolean isClosed() {
	return myClosed;
	}

	@Override
	public synchronized boolean isDirty() {
	return myDirty;
	}

	private synchronized void flush() throws IOException {
	lockStorage();
	try {
	if (myStorage.isDirty() \|\| isDirty()) {
	doFlush();
	}
	}
	finally {
	unlockStorage();
	}
	}

	protected void doFlush() throws IOException {
	markDirty(false);
	myStorage.force();
	}

	@Override
	public synchronized void force() {
	lockStorage();

	try {
	if (myKeyStorage != null) {
	myKeyStorage.force();
	}
	flush();
	}
	catch (IOException e) {
	throw new RuntimeException(e);
	}
	finally {
	unlockStorage();
	}
	}

	protected final void markDirty(boolean dirty) throws IOException {
	//assert Thread.holdsLock(this) \|\| Thread.holdsLock(ourLock); // we hold one lock or another so can access myDirty
	if (dirty && myDirty && !myDirtyStatusUpdateInProgress) return;
	lockStorage();
	try {
	if (myDirty) {
	if (!dirty) {
	myDirtyStatusUpdateInProgress = true;
	if (myMarkCleanCallback != null) myMarkCleanCallback.flush();
	if (!myCorrupted) {
	myStorage.putInt(0, myVersion.correctlyClosedMagic);
	myDirty = false;
	}
	myDirtyStatusUpdateInProgress = false;
	}
	}
	else {
	if (dirty) {
	myDirtyStatusUpdateInProgress = true;
	myStorage.putInt(0, myVersion.dirtyMagic);
	myDirtyStatusUpdateInProgress = false;
	myDirty = true;
	}
	}
	}
	finally {
	unlockStorage();
	}
	}

	protected synchronized void markCorrupted() {
	if (!myCorrupted) {
	myCorrupted = true;
	try {
	markDirty(true);
	force();
	}
	catch (IOException e) {
	// ignore...
	}
	}
	}

	private static class FlyweightKey extends CacheKey {
	public FlyweightKey() {
	super(null, null);
	}

	@Override
	public ShareableKey getStableCopy() {
	return new CacheKey(key, owner);
	}
	}
	}