From a419063dff3e63ce8f59686e7c56a65538bb333c Mon Sep 17 00:00:00 2001 From: Brord van Wierst <brord@iota.org> Date: Mon, 13 May 2019 21:01:45 +0200 Subject: [PATCH 01/16] added recursive weight calc --- .../impl/RecursiveWeightCalculator.java | 125 ++++++++++++++++++ .../impl/CumulativeWeightCalculatorTest.java | 4 +- 2 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java b/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java new file mode 100644 index 0000000000..ca36a03496 --- /dev/null +++ b/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java @@ -0,0 +1,125 @@ +package com.iota.iri.service.tipselection.impl; + +import java.util.ArrayDeque; +import java.util.Collection; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.collections4.CollectionUtils; + +import com.iota.iri.controllers.ApproveeViewModel; +import com.iota.iri.model.Hash; +import com.iota.iri.model.HashId; +import com.iota.iri.model.HashPrefix; +import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.tipselection.RatingCalculator; +import com.iota.iri.storage.Tangle; +import com.iota.iri.utils.collections.impl.TransformingMap; +import com.iota.iri.utils.collections.interfaces.UnIterableMap; + +/** + * Calculates the weight recursively/on the fly instead of building the tree and calculating after + */ +public class RecursiveWeightCalculator implements RatingCalculator { + + public final Tangle tangle; + private final SnapshotProvider snapshotProvider; + + private Map<Hash, Collection<Hash>> txToDirectApprovers = new HashMap<>(); + + /** + * Constructor for Recursive Weight Calculator + * @param tangle Tangle object which acts as a database interface + * @param snapshotProvider accesses ledger's snapshots + */ + public RecursiveWeightCalculator(Tangle tangle, SnapshotProvider snapshotProvider) { + this.tangle = tangle; + this.snapshotProvider = snapshotProvider; + } + + @Override + public UnIterableMap<HashId, Integer> calculate(Hash entryPoint) throws Exception { + Set<Hash> toCheck = new HashSet<>(1); + toCheck.add(entryPoint); + + // Initial capacity of 16, as default for java maps and lists + UnIterableMap<HashId, Integer> hashWeight = createTxHashToCumulativeWeightMap(16); + calculateRatingDfs(entryPoint, hashWeight); + + return hashWeight; + } + + private void calculateRatingDfs(Hash entryPoint, UnIterableMap<HashId, Integer> hashWeight) throws Exception { + Deque<Hash> stack = new ArrayDeque<>(); + Map<Hash, Collection<Hash>> txToDirectApprovers = new HashMap<>(); + + stack.push(entryPoint); + while (CollectionUtils.isNotEmpty(stack)) { + Hash txHash = stack.peek(); + if (!hashWeight.containsKey(txHash)) { + Collection<Hash> appHashes = getTxDirectApproversHashes(txHash, txToDirectApprovers); + if (CollectionUtils.isNotEmpty(appHashes)) { + Hash txApp = getAndRemoveApprover(appHashes); + stack.push(txApp); + continue; + } + } + else { + stack.pop(); + continue; + } + + HashSet<HashId> set = new HashSet<>(); + set.add(txHash); + hashWeight.put(txHash, getRating(txHash, set)); + } + } + + private int getRating(Hash hash, Set<HashId> seenHashes) throws Exception { + int weight = 1; + + Collection<Hash> approvers = getTxDirectApproversHashes(hash, txToDirectApprovers); + for (Hash approver : approvers) { + if (!seenHashes.contains(approver)) { + seenHashes.add(approver); + weight += getRating(approver, seenHashes); + } + } + + return weight; + } + + private Hash getAndRemoveApprover(Collection<Hash> appHashes) { + Iterator<Hash> hashIterator = appHashes.iterator(); + Hash txApp = hashIterator.next(); + hashIterator.remove(); + return txApp; + } + + private Collection<Hash> getTxDirectApproversHashes(Hash txHash, Map<Hash, Collection<Hash>> txToDirectApprovers) + throws Exception { + + Collection<Hash> txApprovers = txToDirectApprovers.get(txHash); + if (txApprovers == null) { + ApproveeViewModel approvers = ApproveeViewModel.load(tangle, txHash); + Collection<Hash> appHashes = CollectionUtils.emptyIfNull(approvers.getHashes()); + txApprovers = new HashSet<>(appHashes.size()); + for (Hash appHash : appHashes) { + //if not genesis (the tx that confirms itself) + if (!snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(appHash)) { + txApprovers.add(appHash); + } + } + txToDirectApprovers.put(txHash, txApprovers); + } + return txApprovers; + } + + private static UnIterableMap<HashId, Integer> createTxHashToCumulativeWeightMap(int size) { + return new TransformingMap<>(size, HashPrefix::createPrefix, null); + } +} diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java index 4f0a1caad8..9539ca69eb 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java @@ -9,6 +9,7 @@ import com.iota.iri.model.HashId; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.service.snapshot.impl.SnapshotProviderImpl; +import com.iota.iri.service.tipselection.RatingCalculator; import com.iota.iri.storage.Tangle; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.collections.interfaces.UnIterableMap; @@ -24,6 +25,7 @@ import static com.iota.iri.TransactionTestUtils.*; + public class CumulativeWeightCalculatorTest { private static final TemporaryFolder dbFolder = new TemporaryFolder(); private static final TemporaryFolder logFolder = new TemporaryFolder(); @@ -31,7 +33,7 @@ public class CumulativeWeightCalculatorTest { "tx%d cumulative weight is not as expected"; private static Tangle tangle; private static SnapshotProvider snapshotProvider; - private static CumulativeWeightCalculator cumulativeWeightCalculator; + private static RatingCalculator cumulativeWeightCalculator; private final Logger log = LoggerFactory.getLogger(this.getClass()); @AfterClass From 855ec6a6040ecc3c33dc7c75504ca406da34a260 Mon Sep 17 00:00:00 2001 From: Brord van Wierst <brord@iota.org> Date: Mon, 13 May 2019 21:03:32 +0200 Subject: [PATCH 02/16] Added ignore to failed test and changed cum-weight test to recursive weight impl --- .../tipselection/impl/CumulativeWeightCalculatorTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java index 9539ca69eb..7b4d90b954 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java @@ -16,6 +16,7 @@ import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; @@ -53,7 +54,7 @@ public static void setUp() throws Exception { tangle.addPersistenceProvider( new RocksDBPersistenceProvider( dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY)); tangle.init(); - cumulativeWeightCalculator = new CumulativeWeightCalculator(tangle, snapshotProvider); + cumulativeWeightCalculator = new RecursiveWeightCalculator(tangle, snapshotProvider); } @Test @@ -271,6 +272,7 @@ public void testTangleWithCircle2() throws Exception { } @Test + @Ignore public void testCollsionsInDiamondTangle() throws Exception { TransactionViewModel transaction, transaction1, transaction2, transaction3; transaction = new TransactionViewModel(getTransactionTrits(), getTransactionHash()); From 9951d392157f4a30635723e4d6bcde24a32bccf3 Mon Sep 17 00:00:00 2001 From: Brord van Wierst <brord@iota.org> Date: Sun, 2 Jun 2019 15:11:39 +0200 Subject: [PATCH 03/16] Changed to queue --- .../tipselection/impl/RecursiveWeightCalculator.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java b/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java index ca36a03496..91a59dbc07 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java @@ -29,7 +29,7 @@ public class RecursiveWeightCalculator implements RatingCalculator { public final Tangle tangle; private final SnapshotProvider snapshotProvider; - private Map<Hash, Collection<Hash>> txToDirectApprovers = new HashMap<>(); + private Map<Hash, ArrayDeque<Hash>> txToDirectApprovers = new HashMap<>(); /** * Constructor for Recursive Weight Calculator @@ -55,7 +55,7 @@ public UnIterableMap<HashId, Integer> calculate(Hash entryPoint) throws Exceptio private void calculateRatingDfs(Hash entryPoint, UnIterableMap<HashId, Integer> hashWeight) throws Exception { Deque<Hash> stack = new ArrayDeque<>(); - Map<Hash, Collection<Hash>> txToDirectApprovers = new HashMap<>(); + Map<Hash, ArrayDeque<Hash>> txToDirectApprovers = new HashMap<>(); stack.push(entryPoint); while (CollectionUtils.isNotEmpty(stack)) { @@ -82,7 +82,7 @@ private void calculateRatingDfs(Hash entryPoint, UnIterableMap<HashId, Integer> private int getRating(Hash hash, Set<HashId> seenHashes) throws Exception { int weight = 1; - Collection<Hash> approvers = getTxDirectApproversHashes(hash, txToDirectApprovers); + ArrayDeque<Hash> approvers = getTxDirectApproversHashes(hash, txToDirectApprovers); for (Hash approver : approvers) { if (!seenHashes.contains(approver)) { seenHashes.add(approver); @@ -100,14 +100,14 @@ private Hash getAndRemoveApprover(Collection<Hash> appHashes) { return txApp; } - private Collection<Hash> getTxDirectApproversHashes(Hash txHash, Map<Hash, Collection<Hash>> txToDirectApprovers) + private ArrayDeque<Hash> getTxDirectApproversHashes(Hash txHash, Map<Hash, ArrayDeque<Hash>> txToDirectApprovers) throws Exception { - Collection<Hash> txApprovers = txToDirectApprovers.get(txHash); + ArrayDeque<Hash> txApprovers = txToDirectApprovers.get(txHash); if (txApprovers == null) { ApproveeViewModel approvers = ApproveeViewModel.load(tangle, txHash); Collection<Hash> appHashes = CollectionUtils.emptyIfNull(approvers.getHashes()); - txApprovers = new HashSet<>(appHashes.size()); + txApprovers = new ArrayDeque<>(appHashes.size()); for (Hash appHash : appHashes) { //if not genesis (the tx that confirms itself) if (!snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(appHash)) { From 676eef4994f91fc627af83c0ac6d36663c5d6b09 Mon Sep 17 00:00:00 2001 From: Brord van Wierst <brord@iota.org> Date: Mon, 3 Jun 2019 19:23:29 +0200 Subject: [PATCH 04/16] Added some map optimizations and decreased db usage --- .../impl/RecursiveWeightCalculator.java | 66 ++++++++++++------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java b/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java index 91a59dbc07..5b5b60fd09 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java @@ -12,6 +12,7 @@ import org.apache.commons.collections4.CollectionUtils; import com.iota.iri.controllers.ApproveeViewModel; +import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; import com.iota.iri.model.HashId; import com.iota.iri.model.HashPrefix; @@ -26,7 +27,7 @@ */ public class RecursiveWeightCalculator implements RatingCalculator { - public final Tangle tangle; + private final Tangle tangle; private final SnapshotProvider snapshotProvider; private Map<Hash, ArrayDeque<Hash>> txToDirectApprovers = new HashMap<>(); @@ -43,17 +44,20 @@ public RecursiveWeightCalculator(Tangle tangle, SnapshotProvider snapshotProvide @Override public UnIterableMap<HashId, Integer> calculate(Hash entryPoint) throws Exception { - Set<Hash> toCheck = new HashSet<>(1); - toCheck.add(entryPoint); - - // Initial capacity of 16, as default for java maps and lists - UnIterableMap<HashId, Integer> hashWeight = createTxHashToCumulativeWeightMap(16); - calculateRatingDfs(entryPoint, hashWeight); + UnIterableMap<HashId, Integer> hashWeight = calculateRatingDfs(entryPoint); return hashWeight; } - private void calculateRatingDfs(Hash entryPoint, UnIterableMap<HashId, Integer> hashWeight) throws Exception { + private UnIterableMap<HashId, Integer> calculateRatingDfs(Hash entryPoint) throws Exception { + TransactionViewModel tvm = TransactionViewModel.fromHash(tangle, entryPoint); + int depth = tvm.snapshotIndex() > 0 + ? snapshotProvider.getLatestSnapshot().getIndex() - tvm.snapshotIndex() + 1 + : 1; + + // Estimated capacity per depth, assumes 5 minute gap in between milestones, at 3tps + UnIterableMap<HashId, Integer> hashWeight = createTxHashToCumulativeWeightMap( 5 * 60 * 3 * depth); + Deque<Hash> stack = new ArrayDeque<>(); Map<Hash, ArrayDeque<Hash>> txToDirectApprovers = new HashMap<>(); @@ -61,28 +65,30 @@ private void calculateRatingDfs(Hash entryPoint, UnIterableMap<HashId, Integer> while (CollectionUtils.isNotEmpty(stack)) { Hash txHash = stack.peek(); if (!hashWeight.containsKey(txHash)) { - Collection<Hash> appHashes = getTxDirectApproversHashes(txHash, txToDirectApprovers); + Collection<Hash> appHashes = getTxDirectApproversHashes(txHash, txToDirectApprovers, txToDirectApprovers); if (CollectionUtils.isNotEmpty(appHashes)) { Hash txApp = getAndRemoveApprover(appHashes); stack.push(txApp); continue; } - } - else { + } else { stack.pop(); continue; } HashSet<HashId> set = new HashSet<>(); set.add(txHash); - hashWeight.put(txHash, getRating(txHash, set)); + + int rating = txHash.equals(entryPoint) ? hashWeight.size() + 1 : getRating(txHash, set); + hashWeight.put(txHash, rating); } + return hashWeight; } private int getRating(Hash hash, Set<HashId> seenHashes) throws Exception { int weight = 1; - ArrayDeque<Hash> approvers = getTxDirectApproversHashes(hash, txToDirectApprovers); + ArrayDeque<Hash> approvers = getTxDirectApproversHashes(hash, txToDirectApprovers, null); for (Hash approver : approvers) { if (!seenHashes.contains(approver)) { seenHashes.add(approver); @@ -100,21 +106,37 @@ private Hash getAndRemoveApprover(Collection<Hash> appHashes) { return txApp; } - private ArrayDeque<Hash> getTxDirectApproversHashes(Hash txHash, Map<Hash, ArrayDeque<Hash>> txToDirectApprovers) + /** + * Finds the approvers of a transaction, and adds it to the txToDirectApprovers map if they werent there yet. + * + * @param txHash The tx we find the approvers of + * @param txToDirectApprovers The map we look in, and add to + * @param fallback The map we check in before going in the database, can be <code>null</code> + * @return + * @throws Exception + */ + private ArrayDeque<Hash> getTxDirectApproversHashes(Hash txHash, + Map<Hash, ArrayDeque<Hash>> txToDirectApprovers, + Map<Hash, ArrayDeque<Hash>> fallback) throws Exception { ArrayDeque<Hash> txApprovers = txToDirectApprovers.get(txHash); if (txApprovers == null) { - ApproveeViewModel approvers = ApproveeViewModel.load(tangle, txHash); - Collection<Hash> appHashes = CollectionUtils.emptyIfNull(approvers.getHashes()); - txApprovers = new ArrayDeque<>(appHashes.size()); - for (Hash appHash : appHashes) { - //if not genesis (the tx that confirms itself) - if (!snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(appHash)) { - txApprovers.add(appHash); + if (fallback != null && fallback.containsKey(txHash)) { + txApprovers = fallback.get(txHash); + txToDirectApprovers.put(txHash, txApprovers); + } else { + ApproveeViewModel approvers = ApproveeViewModel.load(tangle, txHash); + Collection<Hash> appHashes = CollectionUtils.emptyIfNull(approvers.getHashes()); + txApprovers = new ArrayDeque<>(appHashes.size()); + for (Hash appHash : appHashes) { + //if not genesis (the tx that confirms itself) + if (!snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(appHash)) { + txApprovers.add(appHash); + } } + txToDirectApprovers.put(txHash, txApprovers); } - txToDirectApprovers.put(txHash, txApprovers); } return txApprovers; } From 2966d99cb4d0fb81983d932885c8822f5e4bbe2f Mon Sep 17 00:00:00 2001 From: Brord van Wierst <brord@iota.org> Date: Wed, 17 Jul 2019 17:32:00 +0200 Subject: [PATCH 05/16] Implement recursive in codebase --- src/main/java/com/iota/iri/Iota.java | 39 +++++++++++++++++++--------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index 48ed6875fe..78e0269bbe 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -1,5 +1,12 @@ package com.iota.iri; +import java.security.SecureRandom; +import java.util.List; + +import org.apache.commons.lang3.NotImplementedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.iota.iri.conf.IotaConfig; import com.iota.iri.conf.TipSelConfig; import com.iota.iri.controllers.TipsViewModel; @@ -11,7 +18,11 @@ import com.iota.iri.network.replicator.Replicator; import com.iota.iri.service.TipsSolidifier; import com.iota.iri.service.ledger.impl.LedgerServiceImpl; -import com.iota.iri.service.milestone.impl.*; +import com.iota.iri.service.milestone.impl.LatestMilestoneTrackerImpl; +import com.iota.iri.service.milestone.impl.LatestSolidMilestoneTrackerImpl; +import com.iota.iri.service.milestone.impl.MilestoneServiceImpl; +import com.iota.iri.service.milestone.impl.MilestoneSolidifierImpl; +import com.iota.iri.service.milestone.impl.SeenMilestonesRetrieverImpl; import com.iota.iri.service.snapshot.SnapshotException; import com.iota.iri.service.snapshot.impl.LocalSnapshotManagerImpl; import com.iota.iri.service.snapshot.impl.SnapshotProviderImpl; @@ -19,21 +30,25 @@ import com.iota.iri.service.spentaddresses.SpentAddressesException; import com.iota.iri.service.spentaddresses.impl.SpentAddressesProviderImpl; import com.iota.iri.service.spentaddresses.impl.SpentAddressesServiceImpl; -import com.iota.iri.service.tipselection.*; -import com.iota.iri.service.tipselection.impl.*; +import com.iota.iri.service.tipselection.EntryPointSelector; +import com.iota.iri.service.tipselection.RatingCalculator; +import com.iota.iri.service.tipselection.TailFinder; +import com.iota.iri.service.tipselection.TipSelector; +import com.iota.iri.service.tipselection.Walker; +import com.iota.iri.service.tipselection.impl.EntryPointSelectorImpl; +import com.iota.iri.service.tipselection.impl.RecursiveWeightCalculator; +import com.iota.iri.service.tipselection.impl.TailFinderImpl; +import com.iota.iri.service.tipselection.impl.TipSelectorImpl; +import com.iota.iri.service.tipselection.impl.WalkerAlpha; import com.iota.iri.service.transactionpruning.TransactionPruningException; import com.iota.iri.service.transactionpruning.async.AsyncTransactionPruner; -import com.iota.iri.storage.*; +import com.iota.iri.storage.Indexable; +import com.iota.iri.storage.Persistable; +import com.iota.iri.storage.PersistenceProvider; +import com.iota.iri.storage.Tangle; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.Pair; - -import java.security.SecureRandom; -import java.util.List; - import com.iota.iri.zmq.ZmqMessageQueueProvider; -import org.apache.commons.lang3.NotImplementedException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * @@ -300,7 +315,7 @@ private void initializeTangle() { private TipSelector createTipSelector(TipSelConfig config) { EntryPointSelector entryPointSelector = new EntryPointSelectorImpl(tangle, snapshotProvider, latestMilestoneTracker); - RatingCalculator ratingCalculator = new CumulativeWeightCalculator(tangle, snapshotProvider); + RatingCalculator ratingCalculator = new RecursiveWeightCalculator(tangle, snapshotProvider); TailFinder tailFinder = new TailFinderImpl(tangle); Walker walker = new WalkerAlpha(tailFinder, tangle, new SecureRandom(), config); return new TipSelectorImpl(tangle, snapshotProvider, ledgerService, entryPointSelector, ratingCalculator, From 10e3f16d14149c339a291ed0a756fea4d6f6a429 Mon Sep 17 00:00:00 2001 From: Brord van Wierst <brord@iota.org> Date: Wed, 24 Jul 2019 15:54:40 +0200 Subject: [PATCH 06/16] Removed backup cache --- .../impl/RecursiveWeightCalculator.java | 30 +++++++------------ 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java b/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java index 5b5b60fd09..174e6ab3a5 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java @@ -29,8 +29,6 @@ public class RecursiveWeightCalculator implements RatingCalculator { private final Tangle tangle; private final SnapshotProvider snapshotProvider; - - private Map<Hash, ArrayDeque<Hash>> txToDirectApprovers = new HashMap<>(); /** * Constructor for Recursive Weight Calculator @@ -65,7 +63,7 @@ private UnIterableMap<HashId, Integer> calculateRatingDfs(Hash entryPoint) throw while (CollectionUtils.isNotEmpty(stack)) { Hash txHash = stack.peek(); if (!hashWeight.containsKey(txHash)) { - Collection<Hash> appHashes = getTxDirectApproversHashes(txHash, txToDirectApprovers, txToDirectApprovers); + Collection<Hash> appHashes = getTxDirectApproversHashes(txHash, txToDirectApprovers); if (CollectionUtils.isNotEmpty(appHashes)) { Hash txApp = getAndRemoveApprover(appHashes); stack.push(txApp); @@ -88,7 +86,7 @@ private UnIterableMap<HashId, Integer> calculateRatingDfs(Hash entryPoint) throw private int getRating(Hash hash, Set<HashId> seenHashes) throws Exception { int weight = 1; - ArrayDeque<Hash> approvers = getTxDirectApproversHashes(hash, txToDirectApprovers, null); + ArrayDeque<Hash> approvers = getTxDirectApproversHashes(hash, null); for (Hash approver : approvers) { if (!seenHashes.contains(approver)) { seenHashes.add(approver); @@ -116,27 +114,21 @@ private Hash getAndRemoveApprover(Collection<Hash> appHashes) { * @throws Exception */ private ArrayDeque<Hash> getTxDirectApproversHashes(Hash txHash, - Map<Hash, ArrayDeque<Hash>> txToDirectApprovers, - Map<Hash, ArrayDeque<Hash>> fallback) + Map<Hash, ArrayDeque<Hash>> txToDirectApprovers) throws Exception { ArrayDeque<Hash> txApprovers = txToDirectApprovers.get(txHash); if (txApprovers == null) { - if (fallback != null && fallback.containsKey(txHash)) { - txApprovers = fallback.get(txHash); - txToDirectApprovers.put(txHash, txApprovers); - } else { - ApproveeViewModel approvers = ApproveeViewModel.load(tangle, txHash); - Collection<Hash> appHashes = CollectionUtils.emptyIfNull(approvers.getHashes()); - txApprovers = new ArrayDeque<>(appHashes.size()); - for (Hash appHash : appHashes) { - //if not genesis (the tx that confirms itself) - if (!snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(appHash)) { - txApprovers.add(appHash); - } + ApproveeViewModel approvers = ApproveeViewModel.load(tangle, txHash); + Collection<Hash> appHashes = CollectionUtils.emptyIfNull(approvers.getHashes()); + txApprovers = new ArrayDeque<>(appHashes.size()); + for (Hash appHash : appHashes) { + // if not genesis (the tx that confirms itself) + if (!snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(appHash)) { + txApprovers.add(appHash); } - txToDirectApprovers.put(txHash, txApprovers); } + txToDirectApprovers.put(txHash, txApprovers); } return txApprovers; } From ed951f1745e06a3c89d8cc50a8992436d12f6aaa Mon Sep 17 00:00:00 2001 From: Brord van Wierst <brord@iota.org> Date: Thu, 25 Jul 2019 17:08:24 +0200 Subject: [PATCH 07/16] Removed recursiveness --- .../impl/RecursiveWeightCalculator.java | 78 +++++++++---------- .../impl/CumulativeWeightCalculatorTest.java | 35 ++++++--- 2 files changed, 59 insertions(+), 54 deletions(-) diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java b/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java index 174e6ab3a5..8d7d9a308c 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java @@ -5,9 +5,7 @@ import java.util.Deque; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.Map; -import java.util.Set; import org.apache.commons.collections4.CollectionUtils; @@ -55,53 +53,49 @@ private UnIterableMap<HashId, Integer> calculateRatingDfs(Hash entryPoint) throw // Estimated capacity per depth, assumes 5 minute gap in between milestones, at 3tps UnIterableMap<HashId, Integer> hashWeight = createTxHashToCumulativeWeightMap( 5 * 60 * 3 * depth); - - Deque<Hash> stack = new ArrayDeque<>(); - Map<Hash, ArrayDeque<Hash>> txToDirectApprovers = new HashMap<>(); + Map<Hash, HashSet<Hash>> txToDirectApprovers = new HashMap<>(); + + Deque<Hash> stack = new ArrayDeque<>(); stack.push(entryPoint); + while (CollectionUtils.isNotEmpty(stack)) { - Hash txHash = stack.peek(); - if (!hashWeight.containsKey(txHash)) { - Collection<Hash> appHashes = getTxDirectApproversHashes(txHash, txToDirectApprovers); - if (CollectionUtils.isNotEmpty(appHashes)) { - Hash txApp = getAndRemoveApprover(appHashes); - stack.push(txApp); - continue; - } + Hash txHash = stack.peekLast(); + + HashSet<Hash> approvers = getTxDirectApproversHashes(txHash, txToDirectApprovers); + if (null != approvers && (approvers.size() == 0 || hasAll(hashWeight, approvers, stack))) { + approvers.add(txHash); + hashWeight.put(txHash, getRating(approvers, txToDirectApprovers)); + stack.removeLast(); } else { - stack.pop(); - continue; + stack.addAll(approvers); } - - HashSet<HashId> set = new HashSet<>(); - set.add(txHash); - - int rating = txHash.equals(entryPoint) ? hashWeight.size() + 1 : getRating(txHash, set); - hashWeight.put(txHash, rating); } + return hashWeight; } - private int getRating(Hash hash, Set<HashId> seenHashes) throws Exception { - int weight = 1; - - ArrayDeque<Hash> approvers = getTxDirectApproversHashes(hash, null); - for (Hash approver : approvers) { - if (!seenHashes.contains(approver)) { - seenHashes.add(approver); - weight += getRating(approver, seenHashes); + private int getRating(HashSet<Hash> nonDupes, Map<Hash, HashSet<Hash>> txToDirectApprovers) throws Exception { + Deque<Hash> stack = new ArrayDeque<>(nonDupes); + while (CollectionUtils.isNotEmpty(stack)) { + HashSet<Hash> approvers = getTxDirectApproversHashes(stack.pollLast(), txToDirectApprovers); + for (Hash hash : approvers) { + if (nonDupes.add(hash)) { + stack.add(hash); + } } } - - return weight; + + return nonDupes.size(); } - - private Hash getAndRemoveApprover(Collection<Hash> appHashes) { - Iterator<Hash> hashIterator = appHashes.iterator(); - Hash txApp = hashIterator.next(); - hashIterator.remove(); - return txApp; + + private boolean hasAll(UnIterableMap<HashId, Integer> source, HashSet<Hash> requester, Deque<Hash> stack) { + for (Hash h : requester) { + if (!source.containsKey(h) && !stack.contains(h)) { + return false; + } + } + return true; } /** @@ -113,15 +107,14 @@ private Hash getAndRemoveApprover(Collection<Hash> appHashes) { * @return * @throws Exception */ - private ArrayDeque<Hash> getTxDirectApproversHashes(Hash txHash, - Map<Hash, ArrayDeque<Hash>> txToDirectApprovers) + private HashSet<Hash> getTxDirectApproversHashes(Hash txHash, Map<Hash, HashSet<Hash>> txToDirectApprovers) throws Exception { - ArrayDeque<Hash> txApprovers = txToDirectApprovers.get(txHash); + HashSet<Hash> txApprovers = txToDirectApprovers.get(txHash); if (txApprovers == null) { ApproveeViewModel approvers = ApproveeViewModel.load(tangle, txHash); Collection<Hash> appHashes = CollectionUtils.emptyIfNull(approvers.getHashes()); - txApprovers = new ArrayDeque<>(appHashes.size()); + txApprovers = new HashSet<>(appHashes.size()); for (Hash appHash : appHashes) { // if not genesis (the tx that confirms itself) if (!snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(appHash)) { @@ -130,7 +123,8 @@ private ArrayDeque<Hash> getTxDirectApproversHashes(Hash txHash, } txToDirectApprovers.put(txHash, txApprovers); } - return txApprovers; + + return new HashSet<Hash>(txApprovers); } private static UnIterableMap<HashId, Integer> createTxHashToCumulativeWeightMap(int size) { diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java index 7b4d90b954..a8ec1dd1c2 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java @@ -1,6 +1,29 @@ package com.iota.iri.service.tipselection.impl; +import static com.iota.iri.TransactionTestUtils.getTransactionHash; +import static com.iota.iri.TransactionTestUtils.getTransactionTrits; +import static com.iota.iri.TransactionTestUtils.getTransactionTritsWithTrunkAndBranch; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.iota.iri.conf.MainnetConfig; import com.iota.iri.controllers.ApproveeViewModel; import com.iota.iri.controllers.TransactionViewModel; @@ -13,18 +36,6 @@ import com.iota.iri.storage.Tangle; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.collections.interfaces.UnIterableMap; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.*; - -import static com.iota.iri.TransactionTestUtils.*; public class CumulativeWeightCalculatorTest { From 49539550c7c9b5ee26dd91af5713775cf05ee8b1 Mon Sep 17 00:00:00 2001 From: Brord van Wierst <brord@iota.org> Date: Mon, 29 Jul 2019 16:26:28 +0200 Subject: [PATCH 08/16] Review comments --- .../impl/CumulativeWeightCalculator.java | 172 ------------------ .../impl/RecursiveWeightCalculator.java | 64 +++++-- .../impl/CumulativeWeightCalculatorTest.java | 1 - 3 files changed, 44 insertions(+), 193 deletions(-) delete mode 100644 src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java b/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java deleted file mode 100644 index a788e6fc0f..0000000000 --- a/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java +++ /dev/null @@ -1,172 +0,0 @@ -package com.iota.iri.service.tipselection.impl; - -import com.iota.iri.controllers.ApproveeViewModel; -import com.iota.iri.controllers.TransactionViewModel; -import com.iota.iri.model.Hash; -import com.iota.iri.model.HashId; -import com.iota.iri.model.HashPrefix; -import com.iota.iri.service.snapshot.SnapshotProvider; -import com.iota.iri.service.tipselection.RatingCalculator; -import com.iota.iri.utils.collections.impl.TransformingBoundedHashSet; -import com.iota.iri.storage.Tangle; -import com.iota.iri.utils.collections.impl.TransformingMap; -import com.iota.iri.utils.collections.interfaces.BoundedSet; -import com.iota.iri.utils.collections.interfaces.UnIterableMap; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.SetUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.*; - -/** - * Implementation of {@link RatingCalculator} that calculates the cumulative weight - * for each transaction referencing {@code entryPoint}. <br> - * Used to create a weighted random walks. - * - * @see <a href="cumulative.md">https://github.com/alongalky/iota-docs/blob/master/cumulative.md</a> - */ -public class CumulativeWeightCalculator implements RatingCalculator{ - - private static final Logger log = LoggerFactory.getLogger(CumulativeWeightCalculator.class); - private static final int MAX_FUTURE_SET_SIZE = 5000; - - public final Tangle tangle; - private final SnapshotProvider snapshotProvider; - - /** - * Constructor for Cumulative Weight Calculator - * @param tangle Tangle object which acts as a database interface - * @param snapshotProvider acceses ledger's snapshots - */ - public CumulativeWeightCalculator(Tangle tangle, SnapshotProvider snapshotProvider) { - this.tangle = tangle; - this.snapshotProvider = snapshotProvider; - } - - @Override - public UnIterableMap<HashId, Integer> calculate(Hash entryPoint) throws Exception { - log.debug("Start calculating cw starting with tx hash {}", entryPoint); - - LinkedHashSet<Hash> txHashesToRate = sortTransactionsInTopologicalOrder(entryPoint); - return calculateCwInOrder(txHashesToRate); - } - - //Uses DFS algorithm to sort - private LinkedHashSet<Hash> sortTransactionsInTopologicalOrder(Hash startTx) throws Exception { - LinkedHashSet<Hash> sortedTxs = new LinkedHashSet<>(); - Deque<Hash> stack = new ArrayDeque<>(); - Map<Hash, Collection<Hash>> txToDirectApprovers = new HashMap<>(); - - stack.push(startTx); - while (CollectionUtils.isNotEmpty(stack)) { - Hash txHash = stack.peek(); - if (!sortedTxs.contains(txHash)) { - Collection<Hash> appHashes = getTxDirectApproversHashes(txHash, txToDirectApprovers); - if (CollectionUtils.isNotEmpty(appHashes)) { - Hash txApp = getAndRemoveApprover(appHashes); - stack.push(txApp); - continue; - } - } - else { - stack.pop(); - continue; - } - sortedTxs.add(txHash); - } - - return sortedTxs; - } - - private Hash getAndRemoveApprover(Collection<Hash> appHashes) { - Iterator<Hash> hashIterator = appHashes.iterator(); - Hash txApp = hashIterator.next(); - hashIterator.remove(); - return txApp; - } - - private Collection<Hash> getTxDirectApproversHashes(Hash txHash, Map<Hash, Collection<Hash>> txToDirectApprovers) - throws Exception { - Collection<Hash> txApprovers = txToDirectApprovers.get(txHash); - if (txApprovers == null) { - ApproveeViewModel approvers = ApproveeViewModel.load(tangle, txHash); - Collection<Hash> appHashes = CollectionUtils.emptyIfNull(approvers.getHashes()); - txApprovers = new HashSet<>(appHashes.size()); - for (Hash appHash : appHashes) { - //if not genesis (the tx that confirms itself) - if (!snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(appHash)) { - txApprovers.add(appHash); - } - } - txToDirectApprovers.put(txHash, txApprovers); - } - return txApprovers; - } - - //must specify using LinkedHashSet since Java has no interface that guarantees uniqueness and insertion order - private UnIterableMap<HashId, Integer> calculateCwInOrder(LinkedHashSet<Hash> txsToRate) throws Exception { - UnIterableMap<HashId, Set<HashId>> txHashToApprovers = createTxHashToApproversPrefixMap(); - UnIterableMap<HashId, Integer> txHashToCumulativeWeight = createTxHashToCumulativeWeightMap(txsToRate.size()); - - Iterator<Hash> txHashIterator = txsToRate.iterator(); - while (txHashIterator.hasNext()) { - if (Thread.interrupted()) { - throw new InterruptedException(); - } - Hash txHash = txHashIterator.next(); - txHashToCumulativeWeight = updateCw(txHashToApprovers, txHashToCumulativeWeight, txHash); - txHashToApprovers = updateApproversAndReleaseMemory(txHashToApprovers, txHash); - txHashIterator.remove(); - } - return txHashToCumulativeWeight; - } - - - private UnIterableMap<HashId, Set<HashId>> updateApproversAndReleaseMemory(UnIterableMap<HashId, - Set<HashId>> txHashToApprovers, Hash txHash) throws Exception { - Set<HashId> approvers = SetUtils.emptyIfNull(txHashToApprovers.get(txHash)); - - TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(tangle, txHash); - Hash trunkHash = transactionViewModel.getTrunkTransactionHash(); - Hash branchHash = transactionViewModel.getBranchTransactionHash(); - - Set<HashId> trunkApprovers = createApprovers(txHashToApprovers, txHash, approvers, trunkHash); - txHashToApprovers.put(trunkHash, trunkApprovers); - Set<HashId> branchApprovers = createApprovers(txHashToApprovers, txHash, approvers, branchHash); - txHashToApprovers.put(branchHash, branchApprovers); - - txHashToApprovers.remove(txHash); - - return txHashToApprovers; - } - - private Set<HashId> createApprovers(UnIterableMap<HashId, Set<HashId>> txHashToApprovers, HashId txHash, - Set<HashId> approvers, HashId trunkHash) { - Set<HashId> approverSet = createTransformingBoundedSet(approvers); - approverSet.addAll(CollectionUtils.emptyIfNull(txHashToApprovers.get(trunkHash))); - approverSet.add(txHash); - return approverSet; - } - - private static <T extends HashId> UnIterableMap<HashId, Integer> updateCw( - UnIterableMap<HashId, Set<T>> txHashToApprovers, UnIterableMap<HashId, Integer> txToCumulativeWeight, - Hash txHash) { - Set<T> approvers = txHashToApprovers.get(txHash); - int weight = CollectionUtils.emptyIfNull(approvers).size() + 1; - txToCumulativeWeight.put(txHash, weight); - return txToCumulativeWeight; - } - - private static UnIterableMap<HashId, Set<HashId>> createTxHashToApproversPrefixMap() { - return new TransformingMap<>(HashPrefix::createPrefix, null); - } - - private static UnIterableMap<HashId, Integer> createTxHashToCumulativeWeightMap(int size) { - return new TransformingMap<>(size, HashPrefix::createPrefix, null); - } - - private static BoundedSet<HashId> createTransformingBoundedSet(Collection<HashId> c) { - return new TransformingBoundedHashSet<>(c, MAX_FUTURE_SET_SIZE, HashPrefix::createPrefix); - } -} diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java b/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java index 8d7d9a308c..540ffdabd6 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java @@ -2,12 +2,12 @@ import java.util.ArrayDeque; import java.util.Collection; +import java.util.Collections; import java.util.Deque; import java.util.HashMap; import java.util.HashSet; import java.util.Map; - -import org.apache.commons.collections4.CollectionUtils; +import java.util.Set; import com.iota.iri.controllers.ApproveeViewModel; import com.iota.iri.controllers.TransactionViewModel; @@ -21,7 +21,10 @@ import com.iota.iri.utils.collections.interfaces.UnIterableMap; /** - * Calculates the weight recursively/on the fly instead of building the tree and calculating after + * Calculates the weight recursively/on the fly + * Used to create a weighted random walks. + * + * @see <a href="cumulative.md">https://github.com/alongalky/iota-docs/blob/master/cumulative.md</a> */ public class RecursiveWeightCalculator implements RatingCalculator { @@ -40,9 +43,9 @@ public RecursiveWeightCalculator(Tangle tangle, SnapshotProvider snapshotProvide @Override public UnIterableMap<HashId, Integer> calculate(Hash entryPoint) throws Exception { - UnIterableMap<HashId, Integer> hashWeight = calculateRatingDfs(entryPoint); + UnIterableMap<HashId, Integer> hashWeightMap = calculateRatingDfs(entryPoint); - return hashWeight; + return hashWeightMap; } private UnIterableMap<HashId, Integer> calculateRatingDfs(Hash entryPoint) throws Exception { @@ -54,15 +57,15 @@ private UnIterableMap<HashId, Integer> calculateRatingDfs(Hash entryPoint) throw // Estimated capacity per depth, assumes 5 minute gap in between milestones, at 3tps UnIterableMap<HashId, Integer> hashWeight = createTxHashToCumulativeWeightMap( 5 * 60 * 3 * depth); - Map<Hash, HashSet<Hash>> txToDirectApprovers = new HashMap<>(); + Map<Hash, Set<Hash>> txToDirectApprovers = new HashMap<>(); Deque<Hash> stack = new ArrayDeque<>(); stack.push(entryPoint); - while (CollectionUtils.isNotEmpty(stack)) { + while (!stack.isEmpty()) { Hash txHash = stack.peekLast(); - HashSet<Hash> approvers = getTxDirectApproversHashes(txHash, txToDirectApprovers); + Set<Hash> approvers = getTxDirectApproversHashes(txHash, txToDirectApprovers); if (null != approvers && (approvers.size() == 0 || hasAll(hashWeight, approvers, stack))) { approvers.add(txHash); hashWeight.put(txHash, getRating(approvers, txToDirectApprovers)); @@ -75,21 +78,36 @@ private UnIterableMap<HashId, Integer> calculateRatingDfs(Hash entryPoint) throw return hashWeight; } - private int getRating(HashSet<Hash> nonDupes, Map<Hash, HashSet<Hash>> txToDirectApprovers) throws Exception { - Deque<Hash> stack = new ArrayDeque<>(nonDupes); - while (CollectionUtils.isNotEmpty(stack)) { - HashSet<Hash> approvers = getTxDirectApproversHashes(stack.pollLast(), txToDirectApprovers); + /** + * Gets the rating of a set, calculated by checking its approvers + * + * @param startingSet + * @param txToDirectApproversCache + * @return + * @throws Exception + */ + private int getRating(Set<Hash> startingSet, Map<Hash, Set<Hash>> txToDirectApproversCache) throws Exception { + Deque<Hash> stack = new ArrayDeque<>(startingSet); + while (stack.isEmpty()) { + Set<Hash> approvers = getTxDirectApproversHashes(stack.pollLast(), txToDirectApproversCache); for (Hash hash : approvers) { - if (nonDupes.add(hash)) { + if (startingSet.add(hash)) { stack.add(hash); } } } - return nonDupes.size(); + return startingSet.size(); } - private boolean hasAll(UnIterableMap<HashId, Integer> source, HashSet<Hash> requester, Deque<Hash> stack) { + /** + * + * @param source + * @param requester + * @param stack + * @return + */ + private boolean hasAll(UnIterableMap<HashId, Integer> source, Set<Hash> requester, Deque<Hash> stack) { for (Hash h : requester) { if (!source.containsKey(h) && !stack.contains(h)) { return false; @@ -99,21 +117,27 @@ private boolean hasAll(UnIterableMap<HashId, Integer> source, HashSet<Hash> requ } /** - * Finds the approvers of a transaction, and adds it to the txToDirectApprovers map if they werent there yet. + * Finds the approvers of a transaction, and adds it to the txToDirectApprovers map if they weren't there yet. * * @param txHash The tx we find the approvers of * @param txToDirectApprovers The map we look in, and add to * @param fallback The map we check in before going in the database, can be <code>null</code> - * @return + * @return A set with the direct approvers of the given hash * @throws Exception */ - private HashSet<Hash> getTxDirectApproversHashes(Hash txHash, Map<Hash, HashSet<Hash>> txToDirectApprovers) + private Set<Hash> getTxDirectApproversHashes(Hash txHash, Map<Hash, Set<Hash>> txToDirectApprovers) throws Exception { - HashSet<Hash> txApprovers = txToDirectApprovers.get(txHash); + Set<Hash> txApprovers = txToDirectApprovers.get(txHash); if (txApprovers == null) { ApproveeViewModel approvers = ApproveeViewModel.load(tangle, txHash); - Collection<Hash> appHashes = CollectionUtils.emptyIfNull(approvers.getHashes()); + Collection<Hash> appHashes; + if (approvers == null || approvers.getHashes() == null) { + appHashes = Collections.emptySet(); + } else { + appHashes = approvers.getHashes(); + } + txApprovers = new HashSet<>(appHashes.size()); for (Hash appHash : appHashes) { // if not genesis (the tx that confirms itself) diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java index a8ec1dd1c2..61bf7d1090 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java @@ -283,7 +283,6 @@ public void testTangleWithCircle2() throws Exception { } @Test - @Ignore public void testCollsionsInDiamondTangle() throws Exception { TransactionViewModel transaction, transaction1, transaction2, transaction3; transaction = new TransactionViewModel(getTransactionTrits(), getTransactionHash()); From 1ae6a80124effbed01ebc6ecda1a80f5e6bdc216 Mon Sep 17 00:00:00 2001 From: Brord van Wierst <brord@iota.org> Date: Tue, 30 Jul 2019 16:35:01 +0200 Subject: [PATCH 09/16] Renamed and used Cum weight calc, added some javadoc, improved the algorithm --- src/main/java/com/iota/iri/Iota.java | 4 +-- .../tipselection/RatingCalculator.java | 2 +- ...r.java => CumulativeWeightCalculator.java} | 34 ++++++++++++------- .../impl/CumulativeWeightCalculatorTest.java | 3 +- 4 files changed, 25 insertions(+), 18 deletions(-) rename src/main/java/com/iota/iri/service/tipselection/impl/{RecursiveWeightCalculator.java => CumulativeWeightCalculator.java} (79%) diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index 3c7a764321..64d39bbcc0 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -37,8 +37,8 @@ import com.iota.iri.service.tipselection.TailFinder; import com.iota.iri.service.tipselection.TipSelector; import com.iota.iri.service.tipselection.Walker; +import com.iota.iri.service.tipselection.impl.CumulativeWeightCalculator; import com.iota.iri.service.tipselection.impl.EntryPointSelectorImpl; -import com.iota.iri.service.tipselection.impl.RecursiveWeightCalculator; import com.iota.iri.service.tipselection.impl.TailFinderImpl; import com.iota.iri.service.tipselection.impl.TipSelectorImpl; import com.iota.iri.service.tipselection.impl.WalkerAlpha; @@ -345,7 +345,7 @@ private PersistenceProvider createRocksDbProvider(String path, String log, int c private TipSelector createTipSelector(TipSelConfig config) { EntryPointSelector entryPointSelector = new EntryPointSelectorImpl(tangle, snapshotProvider, latestMilestoneTracker); - RatingCalculator ratingCalculator = new RecursiveWeightCalculator(tangle, snapshotProvider); + RatingCalculator ratingCalculator = new CumulativeWeightCalculator(tangle, snapshotProvider); TailFinder tailFinder = new TailFinderImpl(tangle); Walker walker = new WalkerAlpha(tailFinder, tangle, new SecureRandom(), config); return new TipSelectorImpl(tangle, snapshotProvider, ledgerService, entryPointSelector, ratingCalculator, diff --git a/src/main/java/com/iota/iri/service/tipselection/RatingCalculator.java b/src/main/java/com/iota/iri/service/tipselection/RatingCalculator.java index a99f8b01b1..330bef6d5c 100644 --- a/src/main/java/com/iota/iri/service/tipselection/RatingCalculator.java +++ b/src/main/java/com/iota/iri/service/tipselection/RatingCalculator.java @@ -18,7 +18,7 @@ public interface RatingCalculator { * </p> * * @param entryPoint Transaction hash of a selected entry point. - * @return Map of ratings for each transaction that references entryPoint. + * @return Map of ratings for each transaction that references entryPoint. * @throws Exception If DB fails to retrieve transactions */ diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java b/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java similarity index 79% rename from src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java rename to src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java index 540ffdabd6..7374ac0699 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/RecursiveWeightCalculator.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java @@ -26,17 +26,18 @@ * * @see <a href="cumulative.md">https://github.com/alongalky/iota-docs/blob/master/cumulative.md</a> */ -public class RecursiveWeightCalculator implements RatingCalculator { +public class CumulativeWeightCalculator implements RatingCalculator { private final Tangle tangle; private final SnapshotProvider snapshotProvider; /** - * Constructor for Recursive Weight Calculator + * Constructor for Cumulative Weight Calculator + * * @param tangle Tangle object which acts as a database interface * @param snapshotProvider accesses ledger's snapshots */ - public RecursiveWeightCalculator(Tangle tangle, SnapshotProvider snapshotProvider) { + public CumulativeWeightCalculator(Tangle tangle, SnapshotProvider snapshotProvider) { this.tangle = tangle; this.snapshotProvider = snapshotProvider; } @@ -55,40 +56,47 @@ private UnIterableMap<HashId, Integer> calculateRatingDfs(Hash entryPoint) throw : 1; // Estimated capacity per depth, assumes 5 minute gap in between milestones, at 3tps - UnIterableMap<HashId, Integer> hashWeight = createTxHashToCumulativeWeightMap( 5 * 60 * 3 * depth); + UnIterableMap<HashId, Integer> hashWeightMap = createTxHashToCumulativeWeightMap( 5 * 60 * 3 * depth); Map<Hash, Set<Hash>> txToDirectApprovers = new HashMap<>(); Deque<Hash> stack = new ArrayDeque<>(); - stack.push(entryPoint); + stack.addAll(getTxDirectApproversHashes(entryPoint, txToDirectApprovers)); while (!stack.isEmpty()) { Hash txHash = stack.peekLast(); Set<Hash> approvers = getTxDirectApproversHashes(txHash, txToDirectApprovers); - if (null != approvers && (approvers.size() == 0 || hasAll(hashWeight, approvers, stack))) { + if (null != approvers && (approvers.isEmpty() || hasAll(hashWeightMap, approvers, stack))) { + // Add the tx to the approvers list to count itself as +1 weight approvers.add(txHash); - hashWeight.put(txHash, getRating(approvers, txToDirectApprovers)); + + hashWeightMap.put(txHash, getRating(approvers, txToDirectApprovers)); stack.removeLast(); } else { stack.addAll(approvers); } } - return hashWeight; + // If we have a circular reference, its already added, otherwise we save a big calculation + if (!hashWeightMap.containsKey(entryPoint)) { + hashWeightMap.put(entryPoint, hashWeightMap.size() + 1); + } + return hashWeightMap; } /** * Gets the rating of a set, calculated by checking its approvers * - * @param startingSet - * @param txToDirectApproversCache - * @return - * @throws Exception + * @param startingSet All approvers of a certain hash, including the hash itself. + * Should always start with at least 1 hash. + * @param txToDirectApproversCache The cache of approvers, used to prevent double db lookups + * @return The weight, or rating, of the starting hash + * @throws Exception If we can't get the approvers */ private int getRating(Set<Hash> startingSet, Map<Hash, Set<Hash>> txToDirectApproversCache) throws Exception { Deque<Hash> stack = new ArrayDeque<>(startingSet); - while (stack.isEmpty()) { + while (!stack.isEmpty()) { Set<Hash> approvers = getTxDirectApproversHashes(stack.pollLast(), txToDirectApproversCache); for (Hash hash : approvers) { if (startingSet.add(hash)) { diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java index 61bf7d1090..00297fdf73 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java @@ -18,7 +18,6 @@ import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; @@ -65,7 +64,7 @@ public static void setUp() throws Exception { tangle.addPersistenceProvider( new RocksDBPersistenceProvider( dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY)); tangle.init(); - cumulativeWeightCalculator = new RecursiveWeightCalculator(tangle, snapshotProvider); + cumulativeWeightCalculator = new CumulativeWeightCalculator(tangle, snapshotProvider); } @Test From 4d067a180c33705a7e4c15379a803d57076cafde Mon Sep 17 00:00:00 2001 From: Brord van Wierst <brord@iota.org> Date: Tue, 30 Jul 2019 17:09:25 +0200 Subject: [PATCH 10/16] single depth search, removed hasAll --- .../impl/CumulativeWeightCalculator.java | 41 ++++++++----------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java b/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java index 7374ac0699..5eb9e4d0dd 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java @@ -64,18 +64,29 @@ private UnIterableMap<HashId, Integer> calculateRatingDfs(Hash entryPoint) throw stack.addAll(getTxDirectApproversHashes(entryPoint, txToDirectApprovers)); while (!stack.isEmpty()) { - Hash txHash = stack.peekLast(); + Hash txHash = stack.removeLast(); Set<Hash> approvers = getTxDirectApproversHashes(txHash, txToDirectApprovers); - if (null != approvers && (approvers.isEmpty() || hasAll(hashWeightMap, approvers, stack))) { - // Add the tx to the approvers list to count itself as +1 weight + + // If its empty, its a tip! + if (approvers.isEmpty()) { + hashWeightMap.put(txHash, 1); + + // Else we go deeper + } else { + // Add all approvers, given we didnt go there and its not circular + for (Hash h : approvers) { + if (!hashWeightMap.containsKey(h) && !stack.contains(h) && !h.equals(txHash)) { + stack.add(h); + } + } + + // Add the tx to the approvers list to count itself as +1 weight, preventing circular approvers.add(txHash); + // calculate and add rating. Naturally the first time all approvers need to be looked up. Then its cached. hashWeightMap.put(txHash, getRating(approvers, txToDirectApprovers)); - stack.removeLast(); - } else { - stack.addAll(approvers); - } + } } // If we have a circular reference, its already added, otherwise we save a big calculation @@ -107,22 +118,6 @@ private int getRating(Set<Hash> startingSet, Map<Hash, Set<Hash>> txToDirectAppr return startingSet.size(); } - - /** - * - * @param source - * @param requester - * @param stack - * @return - */ - private boolean hasAll(UnIterableMap<HashId, Integer> source, Set<Hash> requester, Deque<Hash> stack) { - for (Hash h : requester) { - if (!source.containsKey(h) && !stack.contains(h)) { - return false; - } - } - return true; - } /** * Finds the approvers of a transaction, and adds it to the txToDirectApprovers map if they weren't there yet. From 1b0aaff2611854dbf4b9dfcfeccd5f0586436f3f Mon Sep 17 00:00:00 2001 From: Brord van Wierst <brord@iota.org> Date: Tue, 30 Jul 2019 17:38:33 +0200 Subject: [PATCH 11/16] Simplified check --- .../tipselection/impl/CumulativeWeightCalculator.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java b/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java index 5eb9e4d0dd..c485ef9538 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java @@ -64,7 +64,7 @@ private UnIterableMap<HashId, Integer> calculateRatingDfs(Hash entryPoint) throw stack.addAll(getTxDirectApproversHashes(entryPoint, txToDirectApprovers)); while (!stack.isEmpty()) { - Hash txHash = stack.removeLast(); + Hash txHash = stack.pollLast(); Set<Hash> approvers = getTxDirectApproversHashes(txHash, txToDirectApprovers); @@ -74,9 +74,9 @@ private UnIterableMap<HashId, Integer> calculateRatingDfs(Hash entryPoint) throw // Else we go deeper } else { - // Add all approvers, given we didnt go there and its not circular + // Add all approvers, given we didnt go there for (Hash h : approvers) { - if (!hashWeightMap.containsKey(h) && !stack.contains(h) && !h.equals(txHash)) { + if (!hashWeightMap.containsKey(h)) { stack.add(h); } } From 29c40a053d389c4c71536a840460d449a7543dc8 Mon Sep 17 00:00:00 2001 From: Brord van Wierst <brord@iota.org> Date: Tue, 30 Jul 2019 17:41:58 +0200 Subject: [PATCH 12/16] Fixed circular -> self referencing --- .../service/tipselection/impl/CumulativeWeightCalculator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java b/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java index c485ef9538..71c8461c12 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java @@ -81,7 +81,7 @@ private UnIterableMap<HashId, Integer> calculateRatingDfs(Hash entryPoint) throw } } - // Add the tx to the approvers list to count itself as +1 weight, preventing circular + // Add the tx to the approvers list to count itself as +1 weight, preventing self-referencing approvers.add(txHash); // calculate and add rating. Naturally the first time all approvers need to be looked up. Then its cached. From babc42f5d048ac0332362ce47c9197e8a39ddf8b Mon Sep 17 00:00:00 2001 From: Brord van Wierst <brord@iota.org> Date: Tue, 30 Jul 2019 17:48:00 +0200 Subject: [PATCH 13/16] Updated description --- .../tipselection/impl/CumulativeWeightCalculator.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java b/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java index 71c8461c12..f5277438a2 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java @@ -21,8 +21,10 @@ import com.iota.iri.utils.collections.interfaces.UnIterableMap; /** - * Calculates the weight recursively/on the fly - * Used to create a weighted random walks. + * Implementation of {@link RatingCalculator} that calculates the cumulative weight + * Calculates the weight recursively/on the fly for each transaction referencing {@code entryPoint}. <br> + * Works using DFS search for new hashes and a BFS calculation. + * Uses cached values to prevent double database lookup for approvers * * @see <a href="cumulative.md">https://github.com/alongalky/iota-docs/blob/master/cumulative.md</a> */ From d410622e6db008b05187e1f237d51a0cdff891b2 Mon Sep 17 00:00:00 2001 From: Brord van Wierst <brord@iota.org> Date: Tue, 30 Jul 2019 18:12:12 +0200 Subject: [PATCH 14/16] Removed link --- .../service/tipselection/impl/CumulativeWeightCalculator.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java b/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java index f5277438a2..8d1c72e765 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java @@ -25,8 +25,6 @@ * Calculates the weight recursively/on the fly for each transaction referencing {@code entryPoint}. <br> * Works using DFS search for new hashes and a BFS calculation. * Uses cached values to prevent double database lookup for approvers - * - * @see <a href="cumulative.md">https://github.com/alongalky/iota-docs/blob/master/cumulative.md</a> */ public class CumulativeWeightCalculator implements RatingCalculator { @@ -91,7 +89,7 @@ private UnIterableMap<HashId, Integer> calculateRatingDfs(Hash entryPoint) throw } } - // If we have a circular reference, its already added, otherwise we save a big calculation + // If we have a self-reference, its already added, otherwise we save a big calculation if (!hashWeightMap.containsKey(entryPoint)) { hashWeightMap.put(entryPoint, hashWeightMap.size() + 1); } From dfa4599bc6262d2b11bfbcc0517f6911cb0d148c Mon Sep 17 00:00:00 2001 From: Brord van Wierst <brord@iota.org> Date: Wed, 31 Jul 2019 04:35:41 +0200 Subject: [PATCH 15/16] Removed HashId need --- .../tipselection/RatingCalculator.java | 6 ++--- .../iota/iri/service/tipselection/Walker.java | 6 ++--- .../impl/CumulativeWeightCalculator.java | 16 +++++------ .../service/tipselection/impl/RatingOne.java | 15 ++++++----- .../tipselection/impl/TipSelectorImpl.java | 23 +++++++++------- .../tipselection/impl/WalkerAlpha.java | 27 +++++++++++-------- 6 files changed, 49 insertions(+), 44 deletions(-) diff --git a/src/main/java/com/iota/iri/service/tipselection/RatingCalculator.java b/src/main/java/com/iota/iri/service/tipselection/RatingCalculator.java index 330bef6d5c..8eeabdcca6 100644 --- a/src/main/java/com/iota/iri/service/tipselection/RatingCalculator.java +++ b/src/main/java/com/iota/iri/service/tipselection/RatingCalculator.java @@ -1,8 +1,8 @@ package com.iota.iri.service.tipselection; +import java.util.Map; + import com.iota.iri.model.Hash; -import com.iota.iri.model.HashId; -import com.iota.iri.utils.collections.interfaces.UnIterableMap; /** * Calculates the rating for a sub graph @@ -22,5 +22,5 @@ public interface RatingCalculator { * @throws Exception If DB fails to retrieve transactions */ - UnIterableMap<HashId, Integer> calculate(Hash entryPoint) throws Exception; + Map<Hash, Integer> calculate(Hash entryPoint) throws Exception; } diff --git a/src/main/java/com/iota/iri/service/tipselection/Walker.java b/src/main/java/com/iota/iri/service/tipselection/Walker.java index a06d4d7e65..116335aa88 100644 --- a/src/main/java/com/iota/iri/service/tipselection/Walker.java +++ b/src/main/java/com/iota/iri/service/tipselection/Walker.java @@ -1,8 +1,8 @@ package com.iota.iri.service.tipselection; +import java.util.Map; + import com.iota.iri.model.Hash; -import com.iota.iri.model.HashId; -import com.iota.iri.utils.collections.interfaces.UnIterableMap; /** * Walks the tangle from an entry point towards tips @@ -24,6 +24,6 @@ public interface Walker { * @return Transaction hash of tip. * @throws Exception If DB fails to retrieve transactions */ - Hash walk(Hash entryPoint, UnIterableMap<HashId, Integer> ratings, WalkValidator walkValidator) throws Exception; + Hash walk(Hash entryPoint, Map<Hash, Integer> ratings, WalkValidator walkValidator) throws Exception; } diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java b/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java index 8d1c72e765..d3416b570a 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java @@ -12,13 +12,9 @@ import com.iota.iri.controllers.ApproveeViewModel; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; -import com.iota.iri.model.HashId; -import com.iota.iri.model.HashPrefix; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.service.tipselection.RatingCalculator; import com.iota.iri.storage.Tangle; -import com.iota.iri.utils.collections.impl.TransformingMap; -import com.iota.iri.utils.collections.interfaces.UnIterableMap; /** * Implementation of {@link RatingCalculator} that calculates the cumulative weight @@ -43,20 +39,20 @@ public CumulativeWeightCalculator(Tangle tangle, SnapshotProvider snapshotProvid } @Override - public UnIterableMap<HashId, Integer> calculate(Hash entryPoint) throws Exception { - UnIterableMap<HashId, Integer> hashWeightMap = calculateRatingDfs(entryPoint); + public Map<Hash, Integer> calculate(Hash entryPoint) throws Exception { + Map<Hash, Integer> hashWeightMap = calculateRatingDfs(entryPoint); return hashWeightMap; } - private UnIterableMap<HashId, Integer> calculateRatingDfs(Hash entryPoint) throws Exception { + private Map<Hash, Integer> calculateRatingDfs(Hash entryPoint) throws Exception { TransactionViewModel tvm = TransactionViewModel.fromHash(tangle, entryPoint); int depth = tvm.snapshotIndex() > 0 ? snapshotProvider.getLatestSnapshot().getIndex() - tvm.snapshotIndex() + 1 : 1; // Estimated capacity per depth, assumes 5 minute gap in between milestones, at 3tps - UnIterableMap<HashId, Integer> hashWeightMap = createTxHashToCumulativeWeightMap( 5 * 60 * 3 * depth); + Map<Hash, Integer> hashWeightMap = createTxHashToCumulativeWeightMap( 5 * 60 * 3 * depth); Map<Hash, Set<Hash>> txToDirectApprovers = new HashMap<>(); @@ -154,7 +150,7 @@ private Set<Hash> getTxDirectApproversHashes(Hash txHash, Map<Hash, Set<Hash>> t return new HashSet<Hash>(txApprovers); } - private static UnIterableMap<HashId, Integer> createTxHashToCumulativeWeightMap(int size) { - return new TransformingMap<>(size, HashPrefix::createPrefix, null); + private static Map<Hash, Integer> createTxHashToCumulativeWeightMap(int size) { + return new HashMap<Hash, Integer>(size); //new TransformingMap<>(size, HashPrefix::createPrefix, null); } } diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/RatingOne.java b/src/main/java/com/iota/iri/service/tipselection/impl/RatingOne.java index d6bcbe1349..162d90c492 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/RatingOne.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/RatingOne.java @@ -1,14 +1,15 @@ package com.iota.iri.service.tipselection.impl; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; +import java.util.Queue; +import java.util.Set; + import com.iota.iri.controllers.ApproveeViewModel; import com.iota.iri.model.Hash; -import com.iota.iri.model.HashId; import com.iota.iri.service.tipselection.RatingCalculator; import com.iota.iri.storage.Tangle; -import com.iota.iri.utils.collections.impl.TransformingMap; -import com.iota.iri.utils.collections.interfaces.UnIterableMap; - -import java.util.*; /** * Implementation of <tt>RatingCalculator</tt> that gives a uniform rating of 1 to each transaction. @@ -23,8 +24,8 @@ public RatingOne(Tangle tangle) { } @Override - public UnIterableMap<HashId, Integer> calculate(Hash entryPoint) throws Exception { - UnIterableMap<HashId, Integer> rating = new TransformingMap<>(null, null); + public Map<Hash, Integer> calculate(Hash entryPoint) throws Exception { + Map<Hash, Integer> rating = new HashMap<>(); Queue<Hash> queue = new LinkedList<>(); queue.add(entryPoint); diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/TipSelectorImpl.java b/src/main/java/com/iota/iri/service/tipselection/impl/TipSelectorImpl.java index 9cdc1d07b0..8393c184b6 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/TipSelectorImpl.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/TipSelectorImpl.java @@ -1,18 +1,21 @@ package com.iota.iri.service.tipselection.impl; +import java.security.InvalidAlgorithmParameterException; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + import com.iota.iri.conf.TipSelConfig; import com.iota.iri.model.Hash; -import com.iota.iri.model.HashId; import com.iota.iri.service.ledger.LedgerService; import com.iota.iri.service.snapshot.SnapshotProvider; -import com.iota.iri.service.tipselection.*; +import com.iota.iri.service.tipselection.EntryPointSelector; +import com.iota.iri.service.tipselection.RatingCalculator; +import com.iota.iri.service.tipselection.TipSelector; +import com.iota.iri.service.tipselection.WalkValidator; +import com.iota.iri.service.tipselection.Walker; import com.iota.iri.storage.Tangle; -import com.iota.iri.utils.collections.interfaces.UnIterableMap; - -import java.security.InvalidAlgorithmParameterException; -import java.util.LinkedList; -import java.util.List; -import java.util.Optional; /** * Implementation of <tt>TipSelector</tt> that selects 2 tips, @@ -89,7 +92,7 @@ public List<Hash> getTransactionsToApprove(int depth, Optional<Hash> reference) //preparation Hash entryPoint = entryPointSelector.getEntryPoint(depth); - UnIterableMap<HashId, Integer> rating = ratingCalculator.calculate(entryPoint); + Map<Hash, Integer> rating = ratingCalculator.calculate(entryPoint); //random walk List<Hash> tips = new LinkedList<>(); @@ -117,7 +120,7 @@ public List<Hash> getTransactionsToApprove(int depth, Optional<Hash> reference) } } - private void checkReference(HashId reference, UnIterableMap<HashId, Integer> rating) + private void checkReference(Hash reference, Map<Hash, Integer> rating) throws InvalidAlgorithmParameterException { if (!rating.containsKey(reference)) { throw new InvalidAlgorithmParameterException(REFERENCE_TRANSACTION_TOO_OLD); diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/WalkerAlpha.java b/src/main/java/com/iota/iri/service/tipselection/impl/WalkerAlpha.java index f2ae75d698..49fda10ad3 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/WalkerAlpha.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/WalkerAlpha.java @@ -1,19 +1,24 @@ package com.iota.iri.service.tipselection.impl; +import java.util.Deque; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Random; +import java.util.Set; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.iota.iri.conf.TipSelConfig; import com.iota.iri.controllers.ApproveeViewModel; import com.iota.iri.model.Hash; -import com.iota.iri.model.HashId; import com.iota.iri.service.tipselection.TailFinder; import com.iota.iri.service.tipselection.WalkValidator; import com.iota.iri.service.tipselection.Walker; import com.iota.iri.storage.Tangle; -import com.iota.iri.utils.collections.interfaces.UnIterableMap; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.*; -import java.util.stream.Collectors; /** * Implementation of <tt>Walker</tt> that performs a weighted random walk @@ -64,7 +69,7 @@ public void setAlpha(double alpha) { } @Override - public Hash walk(Hash entryPoint, UnIterableMap<HashId, Integer> ratings, WalkValidator walkValidator) throws Exception { + public Hash walk(Hash entryPoint, Map<Hash, Integer> ratings, WalkValidator walkValidator) throws Exception { if (!walkValidator.isValid(entryPoint)) { throw new IllegalStateException("entry point failed consistency check: " + entryPoint.toString()); } @@ -88,7 +93,7 @@ public Hash walk(Hash entryPoint, UnIterableMap<HashId, Integer> ratings, WalkVa return traversedTails.getLast(); } - private Optional<Hash> selectApprover(Hash tailHash, UnIterableMap<HashId, Integer> ratings, WalkValidator walkValidator) throws Exception { + private Optional<Hash> selectApprover(Hash tailHash, Map<Hash, Integer> ratings, WalkValidator walkValidator) throws Exception { Set<Hash> approvers = getApprovers(tailHash); return findNextValidTail(ratings, approvers, walkValidator); } @@ -98,7 +103,7 @@ private Set<Hash> getApprovers(Hash tailHash) throws Exception { return approveeViewModel.getHashes(); } - private Optional<Hash> findNextValidTail(UnIterableMap<HashId, Integer> ratings, Set<Hash> approvers, WalkValidator walkValidator) throws Exception { + private Optional<Hash> findNextValidTail(Map<Hash, Integer> ratings, Set<Hash> approvers, WalkValidator walkValidator) throws Exception { Optional<Hash> nextTailHash = Optional.empty(); //select next tail to step to @@ -117,7 +122,7 @@ private Optional<Hash> findNextValidTail(UnIterableMap<HashId, Integer> ratings, return nextTailHash; } - private Optional<Hash> select(UnIterableMap<HashId, Integer> ratings, Set<Hash> approversSet) { + private Optional<Hash> select(Map<Hash, Integer> ratings, Set<Hash> approversSet) { //filter based on tangle state when starting the walk List<Hash> approvers = approversSet.stream().filter(ratings::containsKey).collect(Collectors.toList()); From 3f5d176823eccf86ef76e5f1cceeda0692dc39ba Mon Sep 17 00:00:00 2001 From: Brord van Wierst <brord@iota.org> Date: Wed, 31 Jul 2019 04:46:09 +0200 Subject: [PATCH 16/16] Fixed Transformingmap usage in tests --- .../impl/CumulativeWeightCalculatorTest.java | 18 +++++---- .../tipselection/impl/RatingOneTest.java | 26 +++++++------ .../tipselection/impl/WalkerAlphaTest.java | 38 +++++++++---------- 3 files changed, 42 insertions(+), 40 deletions(-) diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java index 00297fdf73..c2c4cf7249 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java @@ -18,6 +18,7 @@ import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; @@ -34,7 +35,6 @@ import com.iota.iri.service.tipselection.RatingCalculator; import com.iota.iri.storage.Tangle; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; -import com.iota.iri.utils.collections.interfaces.UnIterableMap; public class CumulativeWeightCalculatorTest { @@ -84,7 +84,7 @@ public void testCalculateCumulativeWeight() throws Exception { transaction2.store(tangle, snapshotProvider.getInitialSnapshot()); transaction3.store(tangle, snapshotProvider.getInitialSnapshot()); transaction4.store(tangle, snapshotProvider.getInitialSnapshot()); - UnIterableMap<HashId, Integer> txToCw = cumulativeWeightCalculator.calculate(transaction.getHash()); + Map<Hash, Integer> txToCw = cumulativeWeightCalculator.calculate(transaction.getHash()); Assert.assertEquals(String.format(TX_CUMULATIVE_WEIGHT_IS_NOT_AS_EXPECTED_FORMAT, 4), 1, txToCw.get(transaction4.getHash()).intValue()); @@ -115,7 +115,7 @@ public void testCalculateCumulativeWeightDiamond() throws Exception { log.debug("printing transaction in diamond shape \n {} \n{} {}\n {}", transaction.getHash(), transaction1.getHash(), transaction2.getHash(), transaction3.getHash()); - UnIterableMap<HashId, Integer> txToCw = cumulativeWeightCalculator.calculate(transaction.getHash()); + Map<Hash, Integer> txToCw = cumulativeWeightCalculator.calculate(transaction.getHash()); Assert.assertEquals(String.format(TX_CUMULATIVE_WEIGHT_IS_NOT_AS_EXPECTED_FORMAT, 3), 1, txToCw.get(transaction3.getHash()) @@ -151,7 +151,7 @@ public void testCalculateCumulativeWeightLinear() throws Exception { log.info(String.format("Linear ordered hashes from tip %.4s, %.4s, %.4s, %.4s, %.4s", transaction4.getHash(), transaction3.getHash(), transaction2.getHash(), transaction1.getHash(), transaction.getHash())); - UnIterableMap<HashId, Integer> txToCw = cumulativeWeightCalculator.calculate(transaction.getHash()); + Map<Hash, Integer> txToCw = cumulativeWeightCalculator.calculate(transaction.getHash()); Assert.assertEquals(String.format(TX_CUMULATIVE_WEIGHT_IS_NOT_AS_EXPECTED_FORMAT, 4), @@ -196,7 +196,7 @@ public void testCalculateCumulativeWeight2() throws Exception { transaction.getHash(), transaction1.getHash(), transaction2.getHash(), transaction3.getHash(), transaction4, transaction5, transaction6); - UnIterableMap<HashId, Integer> txToCw = cumulativeWeightCalculator.calculate(transaction.getHash()); + Map<Hash, Integer> txToCw = cumulativeWeightCalculator.calculate(transaction.getHash()); Assert.assertEquals(String.format(TX_CUMULATIVE_WEIGHT_IS_NOT_AS_EXPECTED_FORMAT, 6), 1, txToCw.get(transaction6.getHash()).intValue()); @@ -234,7 +234,7 @@ public void cwCalculationSameAsLegacy() throws Exception { } Map<HashId, Set<HashId>> ratings = new HashMap<>(); updateApproversRecursively(hashes[0], ratings, new HashSet<>()); - UnIterableMap<HashId, Integer> txToCw = cumulativeWeightCalculator.calculate(hashes[0]); + Map<Hash, Integer> txToCw = cumulativeWeightCalculator.calculate(hashes[0]); Assert.assertEquals("missing txs from new calculation", ratings.size(), txToCw.size()); ratings.forEach((hash, weight) -> { @@ -254,7 +254,7 @@ public void testTangleWithCircle() throws Exception { transaction.store(tangle, snapshotProvider.getInitialSnapshot()); - UnIterableMap<HashId, Integer> txToCw = cumulativeWeightCalculator.calculate(transaction.getHash()); + Map<Hash, Integer> txToCw = cumulativeWeightCalculator.calculate(transaction.getHash()); Assert.assertEquals("There should be only one tx in the map", 1, txToCw.size()); Assert.assertEquals("The circle raised the weight", 1, txToCw.get(randomTransactionHash).intValue()); } @@ -281,7 +281,9 @@ public void testTangleWithCircle2() throws Exception { //No infinite loop (which will probably result in an overflow exception) means test has passed } + // Ignored as we do not use HashId in CW, which leads to no more collisions from that. @Test + @Ignore public void testCollsionsInDiamondTangle() throws Exception { TransactionViewModel transaction, transaction1, transaction2, transaction3; transaction = new TransactionViewModel(getTransactionTrits(), getTransactionHash()); @@ -299,7 +301,7 @@ public void testCollsionsInDiamondTangle() throws Exception { log.debug("printing transaction in diamond shape \n {} \n{} {}\n {}", transaction.getHash(), transaction1.getHash(), transaction2.getHash(), transaction3.getHash()); - UnIterableMap<HashId, Integer> txToCw = cumulativeWeightCalculator.calculate(transaction.getHash()); + Map<Hash, Integer> txToCw = cumulativeWeightCalculator.calculate(transaction.getHash()); Assert.assertEquals(String.format(TX_CUMULATIVE_WEIGHT_IS_NOT_AS_EXPECTED_FORMAT, 3), 1, txToCw.get(transaction3.getHash()).intValue()); diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/RatingOneTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/RatingOneTest.java index 8edffdb5a7..6fb0451805 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/RatingOneTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/RatingOneTest.java @@ -1,23 +1,25 @@ package com.iota.iri.service.tipselection.impl; +import static com.iota.iri.TransactionTestUtils.getTransactionHash; +import static com.iota.iri.TransactionTestUtils.getTransactionTrits; +import static com.iota.iri.TransactionTestUtils.getTransactionTritsWithTrunkAndBranch; + +import java.util.Map; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + import com.iota.iri.conf.MainnetConfig; import com.iota.iri.controllers.TransactionViewModel; -import com.iota.iri.model.HashId; +import com.iota.iri.model.Hash; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.service.snapshot.impl.SnapshotProviderImpl; import com.iota.iri.service.tipselection.RatingCalculator; import com.iota.iri.storage.Tangle; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; -import com.iota.iri.utils.collections.interfaces.UnIterableMap; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import static com.iota.iri.TransactionTestUtils.getTransactionTrits; -import static com.iota.iri.TransactionTestUtils.getTransactionHash; -import static com.iota.iri.TransactionTestUtils.getTransactionTritsWithTrunkAndBranch; public class RatingOneTest { private static final TemporaryFolder dbFolder = new TemporaryFolder(); @@ -65,7 +67,7 @@ public void testCalculate() throws Exception { transaction2.store(tangle, snapshotProvider.getInitialSnapshot()); transaction3.store(tangle, snapshotProvider.getInitialSnapshot()); transaction4.store(tangle, snapshotProvider.getInitialSnapshot()); - UnIterableMap<HashId, Integer> rate = rating.calculate(transaction.getHash()); + Map<Hash, Integer> rate = rating.calculate(transaction.getHash()); Assert.assertEquals(TX_CUMULATIVE_WEIGHT_IS_NOT_AS_EXPECTED_FORMAT, 1, rate.get(transaction4.getHash()).intValue()); diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/WalkerAlphaTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/WalkerAlphaTest.java index 703aa70e2e..bbcf43b5b9 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/WalkerAlphaTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/WalkerAlphaTest.java @@ -1,16 +1,8 @@ package com.iota.iri.service.tipselection.impl; -import com.iota.iri.conf.MainnetConfig; -import com.iota.iri.controllers.TransactionViewModel; -import com.iota.iri.model.Hash; -import com.iota.iri.model.HashId; -import com.iota.iri.service.snapshot.SnapshotProvider; -import com.iota.iri.service.snapshot.impl.SnapshotProviderImpl; -import com.iota.iri.service.tipselection.RatingCalculator; -import com.iota.iri.service.tipselection.TailFinder; -import com.iota.iri.storage.Tangle; -import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; -import com.iota.iri.utils.collections.interfaces.UnIterableMap; +import static com.iota.iri.TransactionTestUtils.getTransactionHash; +import static com.iota.iri.TransactionTestUtils.getTransactionTrits; +import static com.iota.iri.TransactionTestUtils.getTransactionTritsWithTrunkAndBranch; import java.util.HashMap; import java.util.Map; @@ -26,9 +18,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static com.iota.iri.TransactionTestUtils.getTransactionTrits; -import static com.iota.iri.TransactionTestUtils.getTransactionHash; -import static com.iota.iri.TransactionTestUtils.getTransactionTritsWithTrunkAndBranch; +import com.iota.iri.conf.MainnetConfig; +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.Hash; +import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.snapshot.impl.SnapshotProviderImpl; +import com.iota.iri.service.tipselection.RatingCalculator; +import com.iota.iri.service.tipselection.TailFinder; +import com.iota.iri.storage.Tangle; +import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; public class WalkerAlphaTest { private static final TemporaryFolder dbFolder = new TemporaryFolder(); @@ -83,7 +81,7 @@ public void testWalkEndsOnlyInRating() throws Exception { //calculate rating RatingCalculator ratingCalculator = new RatingOne(tangle); - UnIterableMap<HashId, Integer> rating = ratingCalculator.calculate(transaction.getHash()); + Map<Hash, Integer> rating = ratingCalculator.calculate(transaction.getHash()); //add 4 after the rating was calculated transaction4 = new TransactionViewModel(getTransactionTritsWithTrunkAndBranch(transaction.getHash(), @@ -120,7 +118,7 @@ public void showWalkDistributionAlphaHalf() throws Exception { //calculate rating RatingCalculator ratingCalculator = new RatingOne(tangle); - UnIterableMap<HashId, Integer> rating = ratingCalculator.calculate(transaction.getHash()); + Map<Hash, Integer> rating = ratingCalculator.calculate(transaction.getHash()); //set a higher rate for transaction2 rating.put(transaction2.getHash(), 10); @@ -163,7 +161,7 @@ public void showWalkDistributionAlphaZero() throws Exception { //calculate rating RatingCalculator ratingCalculator = new RatingOne(tangle); - UnIterableMap<HashId, Integer> rating = ratingCalculator.calculate(transaction.getHash()); + Map<Hash, Integer> rating = ratingCalculator.calculate(transaction.getHash()); //set a higher rate for transaction2 rating.put(transaction2.getHash(), 10); @@ -212,7 +210,7 @@ public void testWalk() throws Exception { //calculate rating RatingCalculator ratingCalculator = new RatingOne(tangle); - UnIterableMap<HashId, Integer> rating = ratingCalculator.calculate(transaction.getHash()); + Map<Hash, Integer> rating = ratingCalculator.calculate(transaction.getHash()); //reach the tips Hash tip = walker.walk(transaction.getHash(), rating, (o -> true)); @@ -239,7 +237,7 @@ public void testWalkDiamond() throws Exception { //calculate rating RatingCalculator ratingCalculator = new RatingOne(tangle); - UnIterableMap<HashId, Integer> rating = ratingCalculator.calculate(transaction.getHash()); + Map<Hash, Integer> rating = ratingCalculator.calculate(transaction.getHash()); //reach the tips Hash tip = walker.walk(transaction.getHash(), rating, (o -> true)); @@ -269,7 +267,7 @@ public void testWalkChain() throws Exception { //calculate rating RatingCalculator ratingCalculator = new RatingOne(tangle); - UnIterableMap<HashId, Integer> rating = ratingCalculator.calculate(transaction.getHash()); + Map<Hash, Integer> rating = ratingCalculator.calculate(transaction.getHash()); //reach the tips Hash tip = walker.walk(transaction.getHash(), rating, (o -> true));