Skip to content

Commit

Permalink
Improve code
Browse files Browse the repository at this point in the history
  • Loading branch information
@ben221199
ben221199 committed Sep 21, 2024
1 parent 894e99d commit 9f063ff
Show file tree
Hide file tree
Showing 4 changed files with 118 additions and 113 deletions.
55 changes: 28 additions & 27 deletions src/main/java/com/lbry/database/BasePrefixDB.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,9 @@ public BasePrefixDB(String path, int maxOpenFiles, String secondaryPath, int max
this.operationStack = new RevertibleOperationStack((byte[] key) -> {
try{
return Optional.of(this.get(key));
}catch(RocksDBException e){}
return Optional.empty();
}catch(RocksDBException e){
return Optional.empty();
}
},(List<byte[]> keys) -> {
List<Optional<byte[]>> optionalKeys = new ArrayList<>();
for(byte[] key : keys){
Expand Down Expand Up @@ -78,8 +79,9 @@ public void unsafeCommit() throws RocksDBException {
}else{
batch.delete(columnFamily,stagedChange.getKey());
}
this.database.write(writeOptions,batch);
}
this.database.write(writeOptions,batch);
batch.close();
}finally{
writeOptions.close();
this.operationStack.clear();
Expand All @@ -92,41 +94,40 @@ public void commit(int height,byte[] blockHash) throws RocksDBException{
List<byte[]> deleteUndos = new ArrayList<>();
if(height>this.maxUndoDepth){
byte[] upperBound = ByteBuffer.allocate(1+8).order(ByteOrder.BIG_ENDIAN).put(Prefix.UNDO.getValue()).putLong(height-this.maxUndoDepth).array();
RocksIterator iterator = this.database.newIterator(new ReadOptions().setIterateUpperBound(new Slice(upperBound)));
ReadOptions readOptions = new ReadOptions().setIterateUpperBound(new Slice(upperBound));
RocksIterator iterator = this.database.newIterator(readOptions);
iterator.seek(ByteBuffer.allocate(1+8).order(ByteOrder.BIG_ENDIAN).put(Prefix.UNDO.getValue()).array());
while(iterator.isValid()){
deleteUndos.add(iterator.key());
iterator.next();
}
readOptions.close();
}
WriteOptions writeOptions = new WriteOptions().setSync(true);
try{
ColumnFamilyHandle undoColumnFamily = this.getColumnFamilyByPrefix(Prefix.UNDO);
WriteOptions writeOptions = new WriteOptions().setSync(true);
try{
WriteBatch batch = new WriteBatch();
for(RevertibleOperation stagedChange : this.operationStack.iterate()){
ColumnFamilyHandle columnFamily = this.getColumnFamilyByPrefix(Prefix.getByValue(stagedChange.getKey()[0]));
if(!stagedChange.isDelete()){
batch.put(columnFamily,stagedChange.getKey(),stagedChange.getValue());
}else{
batch.delete(columnFamily,stagedChange.getKey());
}

}
for(byte[] undoToDelete : deleteUndos){
batch.delete(undoColumnFamily,undoToDelete);
WriteBatch batch = new WriteBatch();
for(RevertibleOperation stagedChange : this.operationStack.iterate()){
ColumnFamilyHandle columnFamily = this.getColumnFamilyByPrefix(Prefix.getByValue(stagedChange.getKey()[0]));
if(!stagedChange.isDelete()){
batch.put(columnFamily,stagedChange.getKey(),stagedChange.getValue());
}else{
batch.delete(columnFamily,stagedChange.getKey());
}
UndoKey undoKey = new UndoKey();
undoKey.height = height;
undoKey.block_hash = blockHash;
byte[] undoKeyBytes = ((PrefixDB)this).undo.packKey(undoKey);
batch.put(undoColumnFamily,undoKeyBytes,undoOperations);
this.database.write(writeOptions,batch);
}finally{
writeOptions.close();
this.operationStack.clear();

}
for(byte[] undoToDelete : deleteUndos){
batch.delete(undoColumnFamily,undoToDelete);
}
final long batchHeight = height;
batch.put(undoColumnFamily,((PrefixDB)this).undo.packKey(new UndoKey(){{
this.block_hash = blockHash;
this.height = batchHeight;
}}),undoOperations);
this.database.write(writeOptions,batch);
batch.close();
}finally{
writeOptions.close();
this.operationStack.clear();
}
}
Expand Down
77 changes: 24 additions & 53 deletions src/main/java/com/lbry/database/revert/RevertibleOperationStack.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.lbry.database.revert;

import com.lbry.database.util.ArrayHelper;
import com.lbry.database.util.MapHelper;
import com.lbry.database.util.Tuple2;

Expand Down Expand Up @@ -53,7 +54,6 @@ public void stashOperations(RevertibleOperation[] operations){
}

public void validateAndApplyStashedOperations(){
// System.err.println("STASH = "+this.stash);
if(this.stash.isEmpty()){
return;
}
Expand All @@ -68,8 +68,7 @@ public void validateAndApplyStashedOperations(){
if(operationArr!=null && operationArr.length>=1 && operation.invert().equals(operationArr[operationArr.length-1])){
this.items.replace(operationArr[0].getKey(),Arrays.copyOfRange(operationArr,0,operationArr.length-1));
continue;
}
if(operationArr!=null && operationArr.length>=1 && operation.equals(operationArr[operationArr.length-1])){
}else if(operationArr!=null && operationArr.length>=1 && operation.equals(operationArr[operationArr.length-1])){
continue;
}else{
needAppend.add(operation);
Expand All @@ -78,63 +77,37 @@ public void validateAndApplyStashedOperations(){
}

Map<byte[],byte[]> existing = new HashMap<>();
// if(this.enforceIntegrity && !uniqueKeys.isEmpty()){
// List<byte[]> uniqueKeysList = new ArrayList<>(uniqueKeys);
// for(int idx=0;idx<uniqueKeys.size();idx+=10000){
// List<byte[]> batch = uniqueKeysList.subList(idx,Math.min(uniqueKeysList.size(),idx+10000));
// Iterator<Optional<byte[]>> iterator = this.multiGet.apply(batch).iterator();
// for(byte[] k : batch){
// byte[] v = iterator.next().get();
// System.err.println(new RevertiblePut(k,v));
// existing.put(k,v);
// }
//
// }
// }
if(this.enforceIntegrity && !uniqueKeys.isEmpty()){
List<byte[]> uniqueKeysList = new ArrayList<>(uniqueKeys);
for(int idx=0;idx<uniqueKeys.size();idx+=10000){
List<byte[]> batch = uniqueKeysList.subList(idx,Math.min(uniqueKeysList.size(),idx+10000));
Iterator<Optional<byte[]>> iterator = this.multiGet.apply(batch).iterator();
for(byte[] k : batch){
Optional<byte[]> vOpt = iterator.next();
vOpt.ifPresent(bytes -> existing.put(k, bytes));
}

}
}

for(RevertibleOperation operation : needAppend){
RevertibleOperation[] operationArr = MapHelper.getValue(this.items,operation.key);
// System.err.println("@ "+operation+ " vs "+Arrays.toString(operationArr));
RevertibleOperation[] operationArr = MapHelper.getValue(this.items,operation.getKey());
if(operationArr!=null && operationArr.length>=1 && operationArr[operationArr.length-1].equals(operation)){
this.items.put(MapHelper.getKey(this.items,operation.getKey()),ArrayHelper.pop(MapHelper.getValue(this.items,operation.getKey())));
RevertibleOperation[] operationArr2 = MapHelper.getValue(this.items,operation.getKey());
List<RevertibleOperation> operationList = Arrays.asList(operationArr2!=null?operationArr2:new RevertibleOperation[0]);
if(!operationList.isEmpty()){
operationList.remove(operationList.size()-1);
}
if(operationList.isEmpty()){
if(operationArr2==null || operationArr2.length==0){
MapHelper.remove(this.items,operation.getKey());
continue;
}
}
if(!this.enforceIntegrity){
RevertibleOperation[] operationArr2 = MapHelper.getValue(this.items,operation.getKey());
List<RevertibleOperation> operationList = Arrays.asList(operationArr2!=null?operationArr2:new RevertibleOperation[0]);
operationList.add(operation);
this.items.put(operationList.get(0).getKey(),operationList.toArray(new RevertibleOperation[0]));
}

RevertibleOperation[] operationArrX = null;
for(Map.Entry<byte[],RevertibleOperation[]> e : this.items.entrySet()){
if(Arrays.equals(e.getKey(),operation.getKey())){
operationArrX = e.getValue();
}
byte[] newKey = MapHelper.getKey(this.items,operation.getKey());
this.items.put(newKey,ArrayHelper.append(MapHelper.getValue(this.items,newKey),operation));
continue;
}

byte[] storedValue = MapHelper.getValue(existing,operation.getKey());
// System.err.println(operation);
boolean hasStoredValue = storedValue!=null;
RevertibleOperation deleteStoredOperation = hasStoredValue?new RevertibleDelete(operation.getKey(),storedValue):null;
boolean deleteStoredOperationInOperationList = false;
if(operationArr!=null){
for(RevertibleOperation o : operationArr){
if(o.equals(deleteStoredOperation)){
deleteStoredOperationInOperationList = true;
break;
}
}
}
boolean willDeleteExistingRecord = deleteStoredOperation!=null && deleteStoredOperationInOperationList;

RevertibleOperation deleteStoredOperation = !hasStoredValue?null:new RevertibleDelete(operation.getKey(),storedValue);
boolean willDeleteExistingRecord = deleteStoredOperation!=null && (MapHelper.getValue(this.items,operation.getKey())!=null);
try{
if(operation.isDelete()){
if(hasStoredValue && !Arrays.equals(storedValue,operation.value) && !willDeleteExistingRecord){
Expand Down Expand Up @@ -167,10 +140,8 @@ public void validateAndApplyStashedOperations(){
throw e;
}
}

RevertibleOperation[] newArr = new RevertibleOperation[operationArrX==null?1:operationArrX.length+1];
newArr[newArr.length-1] = operation;
this.items.put(newArr[0].getKey(),newArr);
byte[] newKey = MapHelper.getKey(this.items,operation.getKey());
this.items.put(newKey,ArrayHelper.append(MapHelper.getValue(this.items,newKey),operation));
}

this.stashedLastOperationForKey.clear();
Expand Down
20 changes: 20 additions & 0 deletions src/main/java/com/lbry/database/util/ArrayHelper.java
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,29 @@
package com.lbry.database.util;

import com.lbry.database.revert.RevertibleOperation;

import java.lang.reflect.Array;
import java.util.Arrays;

public class ArrayHelper{

public static RevertibleOperation[] append(RevertibleOperation[] arr,RevertibleOperation val){
RevertibleOperation[] newArr = new RevertibleOperation[arr!=null?(arr.length+1):1];
if(arr!=null){
System.arraycopy(arr,0,newArr,0,arr.length);
}
newArr[newArr.length-1] = val;
return newArr;
}

public static RevertibleOperation[] pop(RevertibleOperation[] arr){
RevertibleOperation[] newArr = new RevertibleOperation[arr!=null?(arr.length==0?0:arr.length-1):0];
if(arr!=null){
System.arraycopy(arr,0,newArr,0,arr.length-1);
}
return newArr;
}

public static byte[] fill(byte[] arr,byte val){
if(arr!=null){
Arrays.fill(arr,val);
Expand Down
Loading

0 comments on commit 9f063ff

Please sign in to comment.