diff --git a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java
index a2fe3644dc..f73f28e352 100644
--- a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java
+++ b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java
@@ -1,6 +1,5 @@
package com.iota.iri.service.snapshot.impl;
-import com.iota.iri.conf.IotaConfig;
import com.iota.iri.conf.SnapshotConfig;
import com.iota.iri.controllers.ApproveeViewModel;
import com.iota.iri.controllers.MilestoneViewModel;
@@ -23,12 +22,16 @@
import com.iota.iri.utils.log.ProgressLogger;
import com.iota.iri.utils.log.interval.IntervalProgressLogger;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.*;
-import java.util.stream.Collectors;
-
/**
*
* Creates a service instance that allows us to access the business logic for {@link Snapshot}s.
@@ -43,6 +46,36 @@ public class SnapshotServiceImpl implements SnapshotService {
*/
private static final Logger log = LoggerFactory.getLogger(SnapshotServiceImpl.class);
+ /**
+ *
+ * Holds a limit for the amount of milestones we go back in time when generating the solid entry points (to speed up
+ * the snapshot creation).
+ *
+ *
+ * Note: Since the snapshot creation is a "continuous" process where we build upon the information gathered during
+ * the creation of previous snapshots, we do not need to analyze all previous milestones but can rely on
+ * slowly gathering the missing information over time. While this might lead to a situation where the very
+ * first snapshots taken by a node might generate snapshot files that can not reliably be used by other nodes
+ * to sync it is still a reasonable trade-off to reduce the load on the nodes. We just assume that anybody who
+ * wants to share his snapshots with the community as a way to bootstrap new nodes will run his snapshot
+ * enabled node for a few hours before sharing his files (this is a problem in very rare edge cases when
+ * having back-referencing transactions anyway).
+ *
+ */
+ private static final int OUTER_SHELL_SIZE = 100;
+
+ /**
+ *
+ * Maximum age in milestones since creation of solid entry points.
+ *
+ *
+ * Since it is possible to artificially keep old solid entry points alive by periodically attaching new transactions
+ * to them, we limit the life time of solid entry points and ignore them whenever they become too old. This is a
+ * measure against a potential attack vector where somebody might try to blow up the meta data of local snapshots.
+ *
+ */
+ private static final int SOLID_ENTRY_POINT_LIFETIME = 1000;
+
/**
* Holds the tangle object which acts as a database interface.
*/
@@ -56,12 +89,7 @@ public class SnapshotServiceImpl implements SnapshotService {
/**
* Holds the config with important snapshot specific settings.
*/
- private final IotaConfig config;
-
- /**
- * Minimum depth for generating solid entrypoints due to coordinator allowing 15 MS back attachment
- */
- private static final int MIN_LS_DEPTH_MAINNET = 15 + 1;
+ private final SnapshotConfig config;
/**
* Implements the snapshot service. See interface for more information.
@@ -69,7 +97,7 @@ public class SnapshotServiceImpl implements SnapshotService {
* @param snapshotProvider gives us access to the relevant snapshots.
* @param config configuration with snapshot specific settings.
*/
- public SnapshotServiceImpl(Tangle tangle, SnapshotProvider snapshotProvider, IotaConfig config) {
+ public SnapshotServiceImpl(Tangle tangle, SnapshotProvider snapshotProvider, SnapshotConfig config) {
this.tangle = tangle;
this.snapshotProvider = snapshotProvider;
this.config = config;
@@ -250,7 +278,6 @@ public Snapshot generateSnapshot(MilestoneSolidifier milestoneSolidifier, Milest
public Map generateSolidEntryPoints(MilestoneViewModel targetMilestone) throws SnapshotException {
Map solidEntryPoints = new HashMap<>();
solidEntryPoints.put(Hash.NULL_HASH, targetMilestone.index());
- solidEntryPoints.put(targetMilestone.getHash(), targetMilestone.index());
processOldSolidEntryPoints(tangle, snapshotProvider, targetMilestone, solidEntryPoints);
processNewSolidEntryPoints(tangle, snapshotProvider, targetMilestone, solidEntryPoints);
@@ -496,9 +523,100 @@ private void persistLocalSnapshot(SnapshotProvider snapshotProvider, Snapshot ne
snapshotProvider.getInitialSnapshot().update(newSnapshot);
}
-
- private boolean isAboveMinMilestone(int fromIndex, int toIndex) {
- return toIndex - fromIndex <= getSepDepth();
+
+ /**
+ *
+ * This method determines if a transaction is orphaned when none of its approvers is confirmed by a milestone.
+ *
+ *
+ * Since there is no hard definition for when a transaction can be considered to be orphaned, we define orphaned in
+ * relation to a referenceTransaction. If the transaction or any of its direct or indirect approvers saw a
+ * transaction being attached to it, that arrived after our reference transaction, we consider it "not orphaned".
+ *
+ *
+ * Since we currently use milestones as reference transactions that are sufficiently old, this definition in fact is
+ * a relatively safe way to determine if a subtangle "above" a transaction got orphaned.
+ *
+ *
+ * @param tangle Tangle object which acts as a database interface
+ * @param transaction transaction that shall be checked
+ * @param referenceTransaction transaction that acts as a judge to the other transaction
+ * @param processedTransactions transactions that were visited already while trying to determine the orphaned status
+ * @return true if the transaction got orphaned and false otherwise
+ * @throws SnapshotException if anything goes wrong while determining the orphaned status
+ */
+ private boolean isProbablyOrphaned(Tangle tangle, TransactionViewModel transaction,
+ TransactionViewModel referenceTransaction, Set processedTransactions) throws SnapshotException {
+
+ AtomicBoolean nonOrphanedTransactionFound = new AtomicBoolean(false);
+ try {
+ DAGHelper.get(tangle).traverseApprovers(
+ transaction.getHash(),
+ currentTransaction -> !nonOrphanedTransactionFound.get(),
+ currentTransaction -> {
+ if (currentTransaction.getArrivalTime() / 1000L > referenceTransaction.getTimestamp()) {
+ nonOrphanedTransactionFound.set(true);
+ }
+ },
+ processedTransactions
+ );
+ } catch (TraversalException e) {
+ throw new SnapshotException("failed to determine orphaned status of " + transaction, e);
+ }
+
+ return !nonOrphanedTransactionFound.get();
+ }
+
+ /**
+ *
+ * We determine whether future milestones will approve {@param transactionHash}. This should aid in determining
+ * solid entry points.
+ *
+ *
+ * To check if the transaction has non-orphaned approvers we first check if any of its approvers got confirmed by a
+ * future milestone, since this is very cheap. If none of them got confirmed by another milestone we do the more
+ * expensive check from {@link #isProbablyOrphaned(Tangle, TransactionViewModel, TransactionViewModel, Set)}.
+ *
+ *
+ * Since solid entry points have a limited life time and to prevent potential problems due to temporary errors in
+ * the database, we assume that the checked transaction is not orphaned if any error occurs while determining its
+ * status, thus adding solid entry points. This is a storage <=> reliability trade off, since the only bad effect of
+ * having too many solid entry points) is a bigger snapshot file.
+ *
+ *
+ * @param tangle Tangle object which acts as a database interface
+ * @param transactionHash hash of the transaction that shall be checked
+ * @param targetMilestone milestone that is used as an anchor for our checks
+ * @return true if the transaction is a solid entry point and false otherwise
+ */
+ private boolean isNotOrphaned(Tangle tangle, Hash transactionHash, MilestoneViewModel targetMilestone) {
+ Set unconfirmedApprovers = new HashSet<>();
+
+ try {
+ for (Hash approverHash : ApproveeViewModel.load(tangle, transactionHash).getHashes()) {
+ TransactionViewModel approver = TransactionViewModel.fromHash(tangle, approverHash);
+
+ if (approver.snapshotIndex() > targetMilestone.index()) {
+ return true;
+ } else if (approver.snapshotIndex() == 0) {
+ unconfirmedApprovers.add(approver);
+ }
+ }
+
+ Set processedTransactions = new HashSet<>();
+ TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, targetMilestone.getHash());
+ for (TransactionViewModel unconfirmedApprover : unconfirmedApprovers) {
+ if (!isProbablyOrphaned(tangle, unconfirmedApprover, milestoneTransaction, processedTransactions)) {
+ return true;
+ }
+ }
+ } catch (Exception e) {
+ log.error("failed to determine the solid entry point status for transaction " + transactionHash, e);
+
+ return true;
+ }
+
+ return false;
}
/**
@@ -528,7 +646,9 @@ private void processOldSolidEntryPoints(Tangle tangle, SnapshotProvider snapshot
for (Map.Entry solidPoint : orgSolidEntryPoints.entrySet()) {
Hash hash = solidPoint.getKey();
int milestoneIndex = solidPoint.getValue();
- if (!Hash.NULL_HASH.equals(hash) && isAboveMinMilestone(milestoneIndex, targetMilestone.index())) {
+ if (!Hash.NULL_HASH.equals(hash)
+ && targetMilestone.index() - milestoneIndex <= SOLID_ENTRY_POINT_LIFETIME
+ && isNotOrphaned(tangle, hash, targetMilestone)) {
TransactionViewModel tvm = TransactionViewModel.fromHash(tangle, hash);
addTailsToSolidEntryPoints(milestoneIndex, solidEntryPoints, tvm);
solidEntryPoints.put(hash, milestoneIndex);
@@ -549,136 +669,62 @@ private void processOldSolidEntryPoints(Tangle tangle, SnapshotProvider snapshot
* This method retrieves the new solid entry points of the snapshot reference given by the target milestone.
*
*
- * A transaction is considered a solid entry point if it is a
- * bundle tail of a non-orphaned transaction that was approved by a milestone that is above the target milestone.
+ * A transaction is considered a solid entry point if it is a bundle tail that can be traversed down from a
+ * non-orphaned transaction that was approved by a milestone that is above the last local snapshot. Or if it is a
+ * bundle tail of a non-orphaned transaction that was approved by a milestone that is above the last local snapshot.
*
- * It iterates over all relevant unprocessed milestones and analyzes their directly and indirectly approved transactions.
- * Every transaction is checked for being SEP and added to {@param SolidEntryPoints} when required
+ * It iterates over all unprocessed milestones and analyzes their directly and indirectly approved transactions.
+ * Every transaction is checked for being not orphaned and the appropriate SEP is added to {@param SolidEntryPoints}
*
*
+ *
* @param tangle Tangle object which acts as a database interface
* @param snapshotProvider data provider for the {@link Snapshot}s that are relevant for the node
* @param targetMilestone milestone that is used to generate the solid entry points
* @param solidEntryPoints map that is used to collect the solid entry points
* @throws SnapshotException if anything goes wrong while determining the solid entry points
- * @see #getSolidEntryPoints(int, ProgressLogger)
+ * @see #isNotOrphaned(Tangle, Hash, MilestoneViewModel)
*/
private void processNewSolidEntryPoints(Tangle tangle, SnapshotProvider snapshotProvider,
MilestoneViewModel targetMilestone, Map solidEntryPoints) throws SnapshotException {
ProgressLogger progressLogger = new IntervalProgressLogger(
"Taking local snapshot [generating solid entry points]", log);
+
try {
- solidEntryPoints.putAll(getSolidEntryPoints(targetMilestone.index(), progressLogger));
+ progressLogger.start(Math.min(targetMilestone.index() - snapshotProvider.getInitialSnapshot().getIndex(),
+ OUTER_SHELL_SIZE));
+
+ MilestoneViewModel nextMilestone = targetMilestone;
+ while (nextMilestone != null && nextMilestone.index() > snapshotProvider.getInitialSnapshot().getIndex() &&
+ progressLogger.getCurrentStep() < progressLogger.getStepCount()) {
+
+ MilestoneViewModel currentMilestone = nextMilestone;
+ DAGHelper.get(tangle).traverseApprovees(
+ currentMilestone.getHash(),
+ currentTransaction -> currentTransaction.snapshotIndex() >= currentMilestone.index(),
+ currentTransaction -> {
+ if (isNotOrphaned(tangle, currentTransaction.getHash(), targetMilestone)) {
+ addTailsToSolidEntryPoints(targetMilestone.index(), solidEntryPoints,
+ currentTransaction);
+ }
+ }
+ );
+
+ solidEntryPoints.put(currentMilestone.getHash(), targetMilestone.index());
+
+ nextMilestone = MilestoneViewModel.findClosestPrevMilestone(tangle, currentMilestone.index(),
+ snapshotProvider.getInitialSnapshot().getIndex());
+
+ progressLogger.progress();
+ }
+
progressLogger.finish();
} catch (Exception e) {
progressLogger.abort(e);
- throw new SnapshotException("could not generate the solid entry points for " + targetMilestone, e);
- }
- }
-
- /**
- * Generates entrypoints based on target index down to the maximum depth of the node
- *
- * @param targetIndex The milestone index we target to generate entrypoints until.
- * @param progressLogger The logger we use to write progress of entrypoint generation
- * @return a map of entrypoints or null when we were interrupted
- * @throws Exception When we fail to get entry points due to errors generally caused by db interaction
- */
- private Map getSolidEntryPoints(int targetIndex, ProgressLogger progressLogger) throws Exception {
- Map solidEntryPoints = new HashMap<>();
- solidEntryPoints.put(Hash.NULL_HASH, targetIndex);
- log.info("Generating entrypoints for {}", targetIndex);
-
- int sepDepth = getSepDepth();
- // Co back a but below the milestone. Limited to maxDepth or genisis
- int startIndex = Math.max(snapshotProvider.getInitialSnapshot().getIndex(), targetIndex - sepDepth
- ) + 1; // cant start at last snapshot now can we, could be 0!
-
- progressLogger.start(startIndex);
-
- // Iterate from a reasonable old milestone to the target index to check for solid entry points
- for (int milestoneIndex = startIndex; milestoneIndex <= targetIndex; milestoneIndex++) {
- if (Thread.currentThread().isInterrupted()) {
- throw new InterruptedException();
- }
-
- MilestoneViewModel milestone = MilestoneViewModel.get(tangle, milestoneIndex);
- if (milestone == null) {
- log.warn("Failed to find milestone {} during entry point analyzation", milestoneIndex);
- return null;
- }
-
- List approvees = getMilestoneApprovees(milestoneIndex, milestone);
- for (Hash approvee : approvees) {
- if (Thread.currentThread().isInterrupted()) {
- throw new InterruptedException();
- }
-
- if (isSolidEntryPoint(approvee, targetIndex)) {
- // A solid entry point should only be a tail transaction, otherwise the whole bundle can't be reproduced with a snapshot file
- TransactionViewModel tvm = TransactionViewModel.fromHash(tangle, approvee);
- addTailsToSolidEntryPoints(milestoneIndex, solidEntryPoints, tvm);
- }
- }
- progressLogger.progress();
- }
-
- return solidEntryPoints;
- }
- /**
- * Calculates minimum solid entrypoint depth based on network and maxDepth
- *
- * @return The amount of ms we go back under target snapshot for generation of solid entrypoints
- */
- private int getSepDepth() {
- return config.isTestnet() ? config.getMaxDepth() : Math.min(MIN_LS_DEPTH_MAINNET, config.getMaxDepth());
- }
-
- /**
- * isSolidEntryPoint checks whether any direct approver of the given transaction was confirmed
- * by a milestone which is above the target milestone.
- *
- * @param txHash The hash we check as an entrypoint
- * @param targetIndex
- * @return if the transaction is considered a solid entrypoint
- * @throws Exception on db error
- */
- private boolean isSolidEntryPoint(Hash txHash, int targetIndex) throws Exception {
- ApproveeViewModel approvers = ApproveeViewModel.load(tangle, txHash);
- if (approvers.getHashes().isEmpty()) {
- return false;
- }
-
- for (Hash approver : approvers.getHashes()) {
- TransactionViewModel tvm = TransactionViewModel.fromHash(tangle, approver);
- if (tvm != null && tvm.snapshotIndex() > targetIndex) {
- // confirmed by a later milestone than targetIndex => solidEntryPoint
- return true;
- }
+ throw new SnapshotException("could not generate the solid entry points for " + targetMilestone, e);
}
-
- return false;
- }
-
- /**
- * getMilestoneApprovees traverses a milestone and collects all tx that were
- * confirmed by that milestone or higher
- *
- * @param milestoneIndex
- * @param milestone
- * @return
- * @throws TraversalException
- */
- private List getMilestoneApprovees(int milestoneIndex, MilestoneViewModel milestone) throws TraversalException {
- List approvees = new LinkedList<>();
- DAGHelper.get(tangle).traverseApprovees(milestone.getHash(),
- currentTransaction -> currentTransaction.snapshotIndex() == milestoneIndex,
- currentTransaction -> {
- approvees.add(currentTransaction.getHash());
- });
- return approvees;
}
private void addTailsToSolidEntryPoints(int milestoneIndex, Map solidEntryPoints,