diff --git a/src/main/java/com/lbry/database/BasePrefixDB.java b/src/main/java/com/lbry/database/BasePrefixDB.java index 46fb092..e0c3449 100644 --- a/src/main/java/com/lbry/database/BasePrefixDB.java +++ b/src/main/java/com/lbry/database/BasePrefixDB.java @@ -45,8 +45,9 @@ public BasePrefixDB(String path, int maxOpenFiles, String secondaryPath, int max this.operationStack = new RevertibleOperationStack((byte[] key) -> { try{ return Optional.of(this.get(key)); - }catch(RocksDBException e){} - return Optional.empty(); + }catch(RocksDBException e){ + return Optional.empty(); + } },(List keys) -> { List> optionalKeys = new ArrayList<>(); for(byte[] key : keys){ @@ -78,8 +79,9 @@ public void unsafeCommit() throws RocksDBException { }else{ batch.delete(columnFamily,stagedChange.getKey()); } - this.database.write(writeOptions,batch); } + this.database.write(writeOptions,batch); + batch.close(); }finally{ writeOptions.close(); this.operationStack.clear(); @@ -92,41 +94,40 @@ public void commit(int height,byte[] blockHash) throws RocksDBException{ List deleteUndos = new ArrayList<>(); if(height>this.maxUndoDepth){ byte[] upperBound = ByteBuffer.allocate(1+8).order(ByteOrder.BIG_ENDIAN).put(Prefix.UNDO.getValue()).putLong(height-this.maxUndoDepth).array(); - RocksIterator iterator = this.database.newIterator(new ReadOptions().setIterateUpperBound(new Slice(upperBound))); + ReadOptions readOptions = new ReadOptions().setIterateUpperBound(new Slice(upperBound)); + RocksIterator iterator = this.database.newIterator(readOptions); iterator.seek(ByteBuffer.allocate(1+8).order(ByteOrder.BIG_ENDIAN).put(Prefix.UNDO.getValue()).array()); while(iterator.isValid()){ deleteUndos.add(iterator.key()); iterator.next(); } + readOptions.close(); } + WriteOptions writeOptions = new WriteOptions().setSync(true); try{ ColumnFamilyHandle undoColumnFamily = this.getColumnFamilyByPrefix(Prefix.UNDO); - WriteOptions writeOptions = new WriteOptions().setSync(true); - try{ - WriteBatch batch = new WriteBatch(); - for(RevertibleOperation stagedChange : this.operationStack.iterate()){ - ColumnFamilyHandle columnFamily = this.getColumnFamilyByPrefix(Prefix.getByValue(stagedChange.getKey()[0])); - if(!stagedChange.isDelete()){ - batch.put(columnFamily,stagedChange.getKey(),stagedChange.getValue()); - }else{ - batch.delete(columnFamily,stagedChange.getKey()); - } - - } - for(byte[] undoToDelete : deleteUndos){ - batch.delete(undoColumnFamily,undoToDelete); + WriteBatch batch = new WriteBatch(); + for(RevertibleOperation stagedChange : this.operationStack.iterate()){ + ColumnFamilyHandle columnFamily = this.getColumnFamilyByPrefix(Prefix.getByValue(stagedChange.getKey()[0])); + if(!stagedChange.isDelete()){ + batch.put(columnFamily,stagedChange.getKey(),stagedChange.getValue()); + }else{ + batch.delete(columnFamily,stagedChange.getKey()); } - UndoKey undoKey = new UndoKey(); - undoKey.height = height; - undoKey.block_hash = blockHash; - byte[] undoKeyBytes = ((PrefixDB)this).undo.packKey(undoKey); - batch.put(undoColumnFamily,undoKeyBytes,undoOperations); - this.database.write(writeOptions,batch); - }finally{ - writeOptions.close(); - this.operationStack.clear(); + } + for(byte[] undoToDelete : deleteUndos){ + batch.delete(undoColumnFamily,undoToDelete); + } + final long batchHeight = height; + batch.put(undoColumnFamily,((PrefixDB)this).undo.packKey(new UndoKey(){{ + this.block_hash = blockHash; + this.height = batchHeight; + }}),undoOperations); + this.database.write(writeOptions,batch); + batch.close(); }finally{ + writeOptions.close(); this.operationStack.clear(); } } diff --git a/src/main/java/com/lbry/database/revert/RevertibleOperationStack.java b/src/main/java/com/lbry/database/revert/RevertibleOperationStack.java index 098da4b..96322e0 100644 --- a/src/main/java/com/lbry/database/revert/RevertibleOperationStack.java +++ b/src/main/java/com/lbry/database/revert/RevertibleOperationStack.java @@ -1,5 +1,6 @@ package com.lbry.database.revert; +import com.lbry.database.util.ArrayHelper; import com.lbry.database.util.MapHelper; import com.lbry.database.util.Tuple2; @@ -53,7 +54,6 @@ public void stashOperations(RevertibleOperation[] operations){ } public void validateAndApplyStashedOperations(){ -// System.err.println("STASH = "+this.stash); if(this.stash.isEmpty()){ return; } @@ -68,8 +68,7 @@ public void validateAndApplyStashedOperations(){ if(operationArr!=null && operationArr.length>=1 && operation.invert().equals(operationArr[operationArr.length-1])){ this.items.replace(operationArr[0].getKey(),Arrays.copyOfRange(operationArr,0,operationArr.length-1)); continue; - } - if(operationArr!=null && operationArr.length>=1 && operation.equals(operationArr[operationArr.length-1])){ + }else if(operationArr!=null && operationArr.length>=1 && operation.equals(operationArr[operationArr.length-1])){ continue; }else{ needAppend.add(operation); @@ -78,63 +77,37 @@ public void validateAndApplyStashedOperations(){ } Map existing = new HashMap<>(); -// if(this.enforceIntegrity && !uniqueKeys.isEmpty()){ -// List uniqueKeysList = new ArrayList<>(uniqueKeys); -// for(int idx=0;idx batch = uniqueKeysList.subList(idx,Math.min(uniqueKeysList.size(),idx+10000)); -// Iterator> iterator = this.multiGet.apply(batch).iterator(); -// for(byte[] k : batch){ -// byte[] v = iterator.next().get(); -// System.err.println(new RevertiblePut(k,v)); -// existing.put(k,v); -// } -// -// } -// } + if(this.enforceIntegrity && !uniqueKeys.isEmpty()){ + List uniqueKeysList = new ArrayList<>(uniqueKeys); + for(int idx=0;idx batch = uniqueKeysList.subList(idx,Math.min(uniqueKeysList.size(),idx+10000)); + Iterator> iterator = this.multiGet.apply(batch).iterator(); + for(byte[] k : batch){ + Optional vOpt = iterator.next(); + vOpt.ifPresent(bytes -> existing.put(k, bytes)); + } + + } + } for(RevertibleOperation operation : needAppend){ - RevertibleOperation[] operationArr = MapHelper.getValue(this.items,operation.key); -// System.err.println("@ "+operation+ " vs "+Arrays.toString(operationArr)); + RevertibleOperation[] operationArr = MapHelper.getValue(this.items,operation.getKey()); if(operationArr!=null && operationArr.length>=1 && operationArr[operationArr.length-1].equals(operation)){ + this.items.put(MapHelper.getKey(this.items,operation.getKey()),ArrayHelper.pop(MapHelper.getValue(this.items,operation.getKey()))); RevertibleOperation[] operationArr2 = MapHelper.getValue(this.items,operation.getKey()); - List operationList = Arrays.asList(operationArr2!=null?operationArr2:new RevertibleOperation[0]); - if(!operationList.isEmpty()){ - operationList.remove(operationList.size()-1); - } - if(operationList.isEmpty()){ + if(operationArr2==null || operationArr2.length==0){ MapHelper.remove(this.items,operation.getKey()); - continue; } } if(!this.enforceIntegrity){ - RevertibleOperation[] operationArr2 = MapHelper.getValue(this.items,operation.getKey()); - List operationList = Arrays.asList(operationArr2!=null?operationArr2:new RevertibleOperation[0]); - operationList.add(operation); - this.items.put(operationList.get(0).getKey(),operationList.toArray(new RevertibleOperation[0])); - } - - RevertibleOperation[] operationArrX = null; - for(Map.Entry e : this.items.entrySet()){ - if(Arrays.equals(e.getKey(),operation.getKey())){ - operationArrX = e.getValue(); - } + byte[] newKey = MapHelper.getKey(this.items,operation.getKey()); + this.items.put(newKey,ArrayHelper.append(MapHelper.getValue(this.items,newKey),operation)); + continue; } - byte[] storedValue = MapHelper.getValue(existing,operation.getKey()); -// System.err.println(operation); boolean hasStoredValue = storedValue!=null; - RevertibleOperation deleteStoredOperation = hasStoredValue?new RevertibleDelete(operation.getKey(),storedValue):null; - boolean deleteStoredOperationInOperationList = false; - if(operationArr!=null){ - for(RevertibleOperation o : operationArr){ - if(o.equals(deleteStoredOperation)){ - deleteStoredOperationInOperationList = true; - break; - } - } - } - boolean willDeleteExistingRecord = deleteStoredOperation!=null && deleteStoredOperationInOperationList; - + RevertibleOperation deleteStoredOperation = !hasStoredValue?null:new RevertibleDelete(operation.getKey(),storedValue); + boolean willDeleteExistingRecord = deleteStoredOperation!=null && (MapHelper.getValue(this.items,operation.getKey())!=null); try{ if(operation.isDelete()){ if(hasStoredValue && !Arrays.equals(storedValue,operation.value) && !willDeleteExistingRecord){ @@ -167,10 +140,8 @@ public void validateAndApplyStashedOperations(){ throw e; } } - - RevertibleOperation[] newArr = new RevertibleOperation[operationArrX==null?1:operationArrX.length+1]; - newArr[newArr.length-1] = operation; - this.items.put(newArr[0].getKey(),newArr); + byte[] newKey = MapHelper.getKey(this.items,operation.getKey()); + this.items.put(newKey,ArrayHelper.append(MapHelper.getValue(this.items,newKey),operation)); } this.stashedLastOperationForKey.clear(); diff --git a/src/main/java/com/lbry/database/util/ArrayHelper.java b/src/main/java/com/lbry/database/util/ArrayHelper.java index f105f03..0bfd7c3 100644 --- a/src/main/java/com/lbry/database/util/ArrayHelper.java +++ b/src/main/java/com/lbry/database/util/ArrayHelper.java @@ -1,9 +1,29 @@ package com.lbry.database.util; +import com.lbry.database.revert.RevertibleOperation; + +import java.lang.reflect.Array; import java.util.Arrays; public class ArrayHelper{ + public static RevertibleOperation[] append(RevertibleOperation[] arr,RevertibleOperation val){ + RevertibleOperation[] newArr = new RevertibleOperation[arr!=null?(arr.length+1):1]; + if(arr!=null){ + System.arraycopy(arr,0,newArr,0,arr.length); + } + newArr[newArr.length-1] = val; + return newArr; + } + + public static RevertibleOperation[] pop(RevertibleOperation[] arr){ + RevertibleOperation[] newArr = new RevertibleOperation[arr!=null?(arr.length==0?0:arr.length-1):0]; + if(arr!=null){ + System.arraycopy(arr,0,newArr,0,arr.length-1); + } + return newArr; + } + public static byte[] fill(byte[] arr,byte val){ if(arr!=null){ Arrays.fill(arr,val); diff --git a/src/test/java/com/lbry/database/tests/RevertablePrefixDBTest.java b/src/test/java/com/lbry/database/tests/RevertablePrefixDBTest.java index efdbe24..3c2f2a5 100644 --- a/src/test/java/com/lbry/database/tests/RevertablePrefixDBTest.java +++ b/src/test/java/com/lbry/database/tests/RevertablePrefixDBTest.java @@ -1,19 +1,33 @@ package com.lbry.database.tests; import com.lbry.database.PrefixDB; -import com.lbry.database.keys.*; +import com.lbry.database.keys.ActiveAmountKey; +import com.lbry.database.keys.ClaimExpirationKey; +import com.lbry.database.keys.ClaimTakeoverKey; +import com.lbry.database.keys.KeyInterface; +import com.lbry.database.keys.TxNumKey; import com.lbry.database.util.ArrayHelper; -import com.lbry.database.values.*; +import com.lbry.database.values.ActiveAmountValue; +import com.lbry.database.values.ClaimExpirationValue; +import com.lbry.database.values.ClaimTakeoverValue; +import com.lbry.database.values.TxNumValue; +import com.lbry.database.values.DBState; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.file.Files; -import java.util.*; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; import java.util.function.BiFunction; -import org.junit.jupiter.api.*; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; import org.rocksdb.ReadOptions; import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; @@ -33,19 +47,21 @@ public void setUp() throws IOException,RocksDBException{ } @AfterAll - public void tearDown(){} - + public void tearDown(){ + this.database.close(); + this.tmpDir.deleteOnExit(); + } - @Disabled + @Test public void testRollback() throws RocksDBException{ String name = "derp"; - byte[] claim_hash1 = new byte[20]; - Arrays.fill(claim_hash1, (byte) 0x00); - byte[] claim_hash2 = new byte[20]; - Arrays.fill(claim_hash2, (byte) 0x01); - byte[] claim_hash3 = new byte[20]; - Arrays.fill(claim_hash3, (byte) 0x02); + byte[] claimHash1 = new byte[20]; + Arrays.fill(claimHash1, (byte) 0x00); + byte[] claimHash2 = new byte[20]; + Arrays.fill(claimHash2, (byte) 0x01); + byte[] claimHash3 = new byte[20]; + Arrays.fill(claimHash3, (byte) 0x02); int takeoverHeight = 10000000; @@ -55,7 +71,7 @@ public void testRollback() throws RocksDBException{ this.database.claim_takeover.stashPut(new ClaimTakeoverKey(){{ this.normalized_name = name; }},new ClaimTakeoverValue(){{ - this.claim_hash = claim_hash1; + this.claim_hash = claimHash1; this.height = takeoverHeight; }}); assertNull(this.database.claim_takeover.get(new ClaimTakeoverKey(){{ @@ -65,31 +81,27 @@ public void testRollback() throws RocksDBException{ this.normalized_name = name; }})).height); - ///////////////////// - this.database.commit(10000000,ArrayHelper.fill(new byte[32],(byte) 0x00)); assertEquals(10000000,((ClaimTakeoverValue)this.database.claim_takeover.get(new ClaimTakeoverKey(){{ this.normalized_name = name; }})).height); - ///////////////////// - this.database.claim_takeover.stashDelete(new ClaimTakeoverKey(){{ this.normalized_name = name; }},new ClaimTakeoverValue(){{ - this.claim_hash = claim_hash1; + this.claim_hash = claimHash1; this.height = takeoverHeight; }}); this.database.claim_takeover.stashPut(new ClaimTakeoverKey(){{ this.normalized_name = name; }},new ClaimTakeoverValue(){{ - this.claim_hash = claim_hash2; + this.claim_hash = claimHash2; this.height = takeoverHeight + 1; }}); this.database.claim_takeover.stashDelete(new ClaimTakeoverKey(){{ this.normalized_name = name; }},new ClaimTakeoverValue(){{ - this.claim_hash = claim_hash2; + this.claim_hash = claimHash2; this.height = takeoverHeight + 1; }}); this.database.commit(10000001,ArrayHelper.fill(new byte[32],(byte) 0x01)); @@ -99,7 +111,7 @@ public void testRollback() throws RocksDBException{ this.database.claim_takeover.stashPut(new ClaimTakeoverKey(){{ this.normalized_name = name; }},new ClaimTakeoverValue(){{ - this.claim_hash = claim_hash3; + this.claim_hash = claimHash3; this.height = takeoverHeight + 2; }}); this.database.commit(10000002,ArrayHelper.fill(new byte[32],(byte) 0x02)); @@ -107,18 +119,16 @@ public void testRollback() throws RocksDBException{ this.normalized_name = name; }})).height); - ///////////////////// - this.database.claim_takeover.stashDelete(new ClaimTakeoverKey(){{ this.normalized_name = name; }},new ClaimTakeoverValue(){{ - this.claim_hash = claim_hash3; + this.claim_hash = claimHash3; this.height = takeoverHeight + 2; }}); this.database.claim_takeover.stashPut(new ClaimTakeoverKey(){{ this.normalized_name = name; }},new ClaimTakeoverValue(){{ - this.claim_hash = claim_hash2; + this.claim_hash = claimHash2; this.height = takeoverHeight + 3; }}); this.database.commit(10000003,ArrayHelper.fill(new byte[32],(byte) 0x03)); @@ -126,8 +136,6 @@ public void testRollback() throws RocksDBException{ this.normalized_name = name; }})).height); - ///////////////////// - this.database.rollback(10000003,ArrayHelper.fill(new byte[32],(byte) 0x03)); assertEquals(10000002,((ClaimTakeoverValue)this.database.claim_takeover.get(new ClaimTakeoverKey(){{ this.normalized_name = name; @@ -224,7 +232,7 @@ public void testHubDatabaseIterator() throws RocksDBException{ this.genesis = new byte[]{'n','?',(byte) 0xcf,0x12,(byte) 0x99,(byte) 0xd4,(byte) 0xec,']','y',(byte) 0xc3,(byte) 0xa4,(byte) 0xc9,0x1d,'b','J','J',(byte) 0xcf,(byte) 0x9e,'.',0x17,'=',(byte) 0x95,(byte) 0xa1,(byte) 0xa0,'P','O','g','v','i','h','u','V'}; this.height = 0; this.tx_count = 1; - this.tip = new byte[32]; //TODO + this.tip = new byte[]{'V','u','h','i','v','g','O','P',(byte) 0xa0,(byte) 0xa1,(byte) 0x95,'=',0x17,'.',(byte) 0x9e,(byte) 0xcf,'J','J','b',0x1d,(byte) 0xc9,(byte) 0xa4,(byte) 0xc3,'y',']',(byte) 0xec,(byte) 0xd4,(byte) 0x99,0x12,(byte) 0xcf,'?','n'}; this.utxo_flush_count = 1; this.wall_time = 0; this.bit_fields = 1; @@ -251,10 +259,11 @@ public void testHubDatabaseIterator() throws RocksDBException{ } iterator.next(); } + iterator.close(); assertEquals(0,actualMap.size()); } - //TODO (start=98 & stop=99) vs (prefix=98) - //TODO (start=99 & stop=100) vs (prefix=99) + //TODO: (start=98 & stop=99) vs (prefix=98) + //TODO: (start=99 & stop=100) vs (prefix=99) { Map expectedMap = new HashMap<>(); expectedMap.put(this.database.claim_expiration.packKey(new ClaimExpirationKey(){{ @@ -283,6 +292,7 @@ public void testHubDatabaseIterator() throws RocksDBException{ assertArrayEquals(expectedEntry.getKey(),actualEntry.getKey()); assertArrayEquals(expectedEntry.getValue(),actualEntry.getValue()); } + iterator.close(); } { Map expectedMap = new HashMap<>(); @@ -304,12 +314,12 @@ public void testHubDatabaseIterator() throws RocksDBException{ }})); Map actualMap = new HashMap<>(); RocksIterator iterator = this.database.claim_expiration.iterate(); - iterator.seekToLast(); + iterator.seekToFirst(); while(iterator.isValid()){ if(this.database.claim_expiration.unpackKey(iterator.key()).expiration==100){ actualMap.put(iterator.key(),iterator.value()); } - iterator.prev(); // ? + iterator.next(); } assertEquals(expectedMap.size(),actualMap.size()); Iterator> expectedEntrySetIterator = expectedMap.entrySet().iterator(); @@ -320,6 +330,7 @@ public void testHubDatabaseIterator() throws RocksDBException{ assertArrayEquals(expectedEntry.getKey(),actualEntry.getKey()); assertArrayEquals(expectedEntry.getValue(),actualEntry.getValue()); } + iterator.close(); } //TODO (start=100 & stop=101) vs (prefix=100) { @@ -350,6 +361,7 @@ public void testHubDatabaseIterator() throws RocksDBException{ assertArrayEquals(expectedEntry.getKey(),actualEntry.getKey()); assertArrayEquals(expectedEntry.getValue(),actualEntry.getValue()); } + iterator.close(); } { Map expectedMap = new HashMap<>(); @@ -379,6 +391,7 @@ public void testHubDatabaseIterator() throws RocksDBException{ assertArrayEquals(expectedEntry.getKey(),actualEntry.getKey()); assertArrayEquals(expectedEntry.getValue(),actualEntry.getValue()); } + iterator.close(); } }