From 5b369101f002aad824e62f22e803f908fc2ee282 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Fri, 4 Aug 2023 19:13:50 +0800 Subject: [PATCH 1/9] initial implementation of closest n destinations --- .../java/com/conveyal/r5/OneOriginResult.java | 11 +++- .../conveyal/r5/analyst/NearestNResult.java | 60 +++++++++++++++++++ .../r5/analyst/TravelTimeReducer.java | 11 +++- .../r5/analyst/cluster/AnalysisWorker.java | 2 +- .../analyst/cluster/AnalysisWorkerTask.java | 3 + .../analyst/cluster/RegionalWorkResult.java | 8 +++ 6 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 src/main/java/com/conveyal/r5/analyst/NearestNResult.java diff --git a/src/main/java/com/conveyal/r5/OneOriginResult.java b/src/main/java/com/conveyal/r5/OneOriginResult.java index 6c721d41e..24e9348b4 100644 --- a/src/main/java/com/conveyal/r5/OneOriginResult.java +++ b/src/main/java/com/conveyal/r5/OneOriginResult.java @@ -1,6 +1,7 @@ package com.conveyal.r5; import com.conveyal.r5.analyst.AccessibilityResult; +import com.conveyal.r5.analyst.NearestNResult; import com.conveyal.r5.analyst.cluster.PathResult; import com.conveyal.r5.analyst.cluster.TravelTimeResult; @@ -20,10 +21,18 @@ public class OneOriginResult { public final PathResult paths; - public OneOriginResult(TravelTimeResult travelTimes, AccessibilityResult accessibility, PathResult paths) { + public final NearestNResult nearest; + + public OneOriginResult( + TravelTimeResult travelTimes, + AccessibilityResult accessibility, + PathResult paths, + NearestNResult nearest + ) { this.travelTimes = travelTimes; this.accessibility = accessibility; this.paths = paths; + this.nearest = nearest; } } diff --git a/src/main/java/com/conveyal/r5/analyst/NearestNResult.java b/src/main/java/com/conveyal/r5/analyst/NearestNResult.java new file mode 100644 index 000000000..6f37060a5 --- /dev/null +++ b/src/main/java/com/conveyal/r5/analyst/NearestNResult.java @@ -0,0 +1,60 @@ +package com.conveyal.r5.analyst; + +import com.conveyal.r5.analyst.cluster.AnalysisWorkerTask; + +/** + * An instance of this is included in a OneOriginResult for reporting the nearest N destinations. + * If we use more than one destination point set they must all be aligned with the same number of destinations. + */ +public class NearestNResult { + + private final PointSet[] destinationPointSets; + private final int nPercentiles; + private final int nOpportunities; + + public static class NearbyOpportunity { + public int seconds = Integer.MAX_VALUE; + public int target; + public String id; + } + + // For each percentile, the closest N destinations + public final NearbyOpportunity[][] opportunities; + + public NearestNResult (AnalysisWorkerTask task) { + this.destinationPointSets = task.destinationPointSets; + this.nPercentiles = task.percentiles.length; + this.nOpportunities = 3; + this.opportunities = new NearbyOpportunity[nPercentiles][nOpportunities]; + } + + public void record (int target, int[] travelTimePercentilesSeconds) { + for (int p = 0; p < nPercentiles; p++) { + // Find the slot with the highest travel time less than the reported travel time for this percentile. + int toReplace = -1; + for (int i = 0; i < nOpportunities; i++) { + if (opportunities[p][i] == null) { + opportunities[p][i] = new NearbyOpportunity(); + toReplace = i; + break; + } + if (travelTimePercentilesSeconds[p] < opportunities[p][i].seconds) { + if (toReplace < 0 || (opportunities[p][i].seconds > opportunities[p][toReplace].seconds)) { + toReplace = i; + } + } + } + if (toReplace >= 0) { + opportunities[p][toReplace].seconds = travelTimePercentilesSeconds[p]; + opportunities[p][toReplace].target = target; + // There are actually as many IDs as there are destination point sets but record only the first for now. + // However they are only set in regional analyses. + if (destinationPointSets != null) { + opportunities[p][toReplace].id = destinationPointSets[0].getId(target); + } + } + } + } + + +} diff --git a/src/main/java/com/conveyal/r5/analyst/TravelTimeReducer.java b/src/main/java/com/conveyal/r5/analyst/TravelTimeReducer.java index 4e1ca30bc..957b71646 100644 --- a/src/main/java/com/conveyal/r5/analyst/TravelTimeReducer.java +++ b/src/main/java/com/conveyal/r5/analyst/TravelTimeReducer.java @@ -46,6 +46,9 @@ public class TravelTimeReducer { /** Retains the paths to one or all destinations, for recording in CSV or reporting in the UI. */ private PathResult pathResult = null; + /** Retains the nearest N destinations from this origin. */ + private NearestNResult nearestNResult = null; + /** If we are calculating accessibility, the PointSets containing opportunities. */ private PointSet[] destinationPointSets; @@ -140,6 +143,9 @@ public TravelTimeReducer (AnalysisWorkerTask task, TransportNetwork network) { if (task.includePathResults) { pathResult = new PathResult(task, network.transitLayer); } + if (task.includeNearestN) { + nearestNResult = new NearestNResult(task); + } // Validate and copy the travel time cutoffs, converting them to seconds to avoid repeated multiplication // in tight loops. Also find the points where the decay function reaches zero for these cutoffs. @@ -271,6 +277,9 @@ private void recordTravelTimePercentilesForTarget (int target, int[] travelTimeP } } } + if (nearestNResult != null) { + nearestNResult.record(target, travelTimePercentilesSeconds); + } } /** @@ -329,7 +338,7 @@ private int convertToMinutes (int timeSeconds) { * origin point is not connected to the street network. */ public OneOriginResult finish () { - return new OneOriginResult(travelTimeResult, accessibilityResult, pathResult); + return new OneOriginResult(travelTimeResult, accessibilityResult, pathResult, nearestNResult); } /** diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java index aad7da99d..69839d5ab 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java @@ -457,7 +457,7 @@ protected void handleOneRegionalTask (RegionalTask task) throws Throwable { // progress. This avoids crashing the backend by sending back massive 2 million element travel times // that have already been written to S3, and throwing exceptions on old backends that can't deal with // null AccessibilityResults. - oneOriginResult = new OneOriginResult(null, new AccessibilityResult(task), null); + oneOriginResult = new OneOriginResult(null, new AccessibilityResult(task), null, null); } // Accumulate accessibility results, which will be returned to the backend in batches. diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorkerTask.java b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorkerTask.java index d1e88f78c..15cd408ac 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorkerTask.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorkerTask.java @@ -99,6 +99,9 @@ public abstract class AnalysisWorkerTask extends ProfileRequest { */ public boolean includePathResults = false; + /** Whether to record the n closest destination points to the origin. */ + public boolean includeNearestN = false; + /** Whether to build a histogram of travel times to each destination, generally used in testing and debugging. */ public boolean recordTravelTimeHistograms = false; diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java b/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java index 39a650d3e..0378012f7 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java @@ -1,6 +1,7 @@ package com.conveyal.r5.analyst.cluster; import com.conveyal.r5.OneOriginResult; +import com.conveyal.r5.analyst.NearestNResult.NearbyOpportunity; import com.conveyal.r5.util.ExceptionUtils; import java.util.ArrayList; @@ -37,6 +38,12 @@ public class RegionalWorkResult { */ public int[][][] accessibilityValues; + /** + * The nearest n destinations for each percentile of travel time. + * Each item contains a travel time, target index, and ID. + */ + public NearbyOpportunity[][] nearestN; + /** * If this field is non-null, the worker is reporting an error that compromises the quality of the result at this * origin point, and potentially for then entire regional analysis. Put into a Set on backend since all workers @@ -59,6 +66,7 @@ public RegionalWorkResult(OneOriginResult result, RegionalTask task) { this.travelTimeValues = result.travelTimes == null ? null : result.travelTimes.values; this.accessibilityValues = result.accessibility == null ? null : result.accessibility.getIntValues(); this.pathResult = result.paths == null ? null : result.paths.summarizeIterations(PathResult.Stat.MINIMUM); + this.nearestN = result.nearest == null ? null : result.nearest.opportunities; // TODO checkTravelTimeInvariants, checkAccessibilityInvariants to verify that values are monotonically increasing } From b0a0a8ea86edb43d6e2d1ed7e51155fb216f3735 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Mon, 7 Aug 2023 01:43:21 +0800 Subject: [PATCH 2/9] record opportunities per minute (discrete derivative of accessibility) still no way to collate and export results for a regional analysis --- .../conveyal/r5/analyst/NearestNResult.java | 64 ++++++++++++++----- .../r5/analyst/TravelTimeReducer.java | 7 +- .../r5/analyst/cluster/AnalysisWorker.java | 36 ++++++++--- .../analyst/cluster/AnalysisWorkerTask.java | 3 - .../analyst/cluster/RegionalWorkResult.java | 11 +++- .../r5/analyst/cluster/TravelTimeResult.java | 3 + 6 files changed, 90 insertions(+), 34 deletions(-) diff --git a/src/main/java/com/conveyal/r5/analyst/NearestNResult.java b/src/main/java/com/conveyal/r5/analyst/NearestNResult.java index 6f37060a5..1df5a6b8d 100644 --- a/src/main/java/com/conveyal/r5/analyst/NearestNResult.java +++ b/src/main/java/com/conveyal/r5/analyst/NearestNResult.java @@ -1,13 +1,19 @@ package com.conveyal.r5.analyst; import com.conveyal.r5.analyst.cluster.AnalysisWorkerTask; +import com.google.common.base.Preconditions; + +import static com.conveyal.r5.common.Util.notNullOrEmpty; /** * An instance of this is included in a OneOriginResult for reporting the nearest N destinations. * If we use more than one destination point set they must all be aligned with the same number of destinations. + * + * This supports two kinds of results... (expand from comments on https://github.com/conveyal/r5/pull/884) */ public class NearestNResult { + private static final int DEFAULT_N_OPPORTUNITIES = 3; private final PointSet[] destinationPointSets; private final int nPercentiles; private final int nOpportunities; @@ -18,43 +24,69 @@ public static class NearbyOpportunity { public String id; } - // For each percentile, the closest N destinations - public final NearbyOpportunity[][] opportunities; + // Fields to accumulate results + + /** For each percentile, the closest N destinations */ + public final NearbyOpportunity[][] nearby; + + /** + * For each destination set, for each percentile, for each minute of travel from 0 to 120, the number of + * opportunities reached in travel times from i (inclusive) to i+1 (exclusive). + */ + public final double[][][] opportunitiesPerMinute; public NearestNResult (AnalysisWorkerTask task) { + Preconditions.checkArgument( + notNullOrEmpty(task.destinationPointSets), + "Dual accessibility requires at least one destination pointset." + ); this.destinationPointSets = task.destinationPointSets; this.nPercentiles = task.percentiles.length; - this.nOpportunities = 3; - this.opportunities = new NearbyOpportunity[nPercentiles][nOpportunities]; + this.nOpportunities = DEFAULT_N_OPPORTUNITIES; + this.nearby = new NearbyOpportunity[nPercentiles][nOpportunities]; + this.opportunitiesPerMinute = new double[destinationPointSets.length][nPercentiles][120]; } public void record (int target, int[] travelTimePercentilesSeconds) { + // Incremenet histogram bin for the number of minutes of travel, by the number of opportunities at the target. + for (int d = 0; d < destinationPointSets.length; d++) { + PointSet dps = destinationPointSets[d]; + for (int p = 0; p < nPercentiles; p++) { + int i = travelTimePercentilesSeconds[p] / 60; + if (i <= 120) { + opportunitiesPerMinute[d][p][i] += dps.getOpportunityCount(target); + } + } + } + // Insert the destination in the list of nearest destinations if it improves on those already present. + // Nearest N only makes sense for freeform point sets that define IDs for each point. If there were + // multiple destination point sets, each of them could define a different ID for the same point. + // However unlike gridded pointsets, we don't allow more than one freeform pointset in the same request. + // Note also that in single-point analyses, the destination pointsets may be null if the user has not + // selected any pointset in the UI (or if the step function is selected rather than another decay function). + // So if any destinations are present, we only care about the first set. for (int p = 0; p < nPercentiles; p++) { // Find the slot with the highest travel time less than the reported travel time for this percentile. + // TODO keep ordered by scanning from high end and shifting all elements to insert. int toReplace = -1; for (int i = 0; i < nOpportunities; i++) { - if (opportunities[p][i] == null) { - opportunities[p][i] = new NearbyOpportunity(); + if (nearby[p][i] == null) { + nearby[p][i] = new NearbyOpportunity(); toReplace = i; break; } - if (travelTimePercentilesSeconds[p] < opportunities[p][i].seconds) { - if (toReplace < 0 || (opportunities[p][i].seconds > opportunities[p][toReplace].seconds)) { + if (travelTimePercentilesSeconds[p] < nearby[p][i].seconds) { + if (toReplace < 0 || (nearby[p][i].seconds > nearby[p][toReplace].seconds)) { toReplace = i; } } } if (toReplace >= 0) { - opportunities[p][toReplace].seconds = travelTimePercentilesSeconds[p]; - opportunities[p][toReplace].target = target; - // There are actually as many IDs as there are destination point sets but record only the first for now. - // However they are only set in regional analyses. - if (destinationPointSets != null) { - opportunities[p][toReplace].id = destinationPointSets[0].getId(target); - } + nearby[p][toReplace].seconds = travelTimePercentilesSeconds[p]; + nearby[p][toReplace].target = target; + nearby[p][toReplace].id = destinationPointSets[0].getId(target); } } } - } diff --git a/src/main/java/com/conveyal/r5/analyst/TravelTimeReducer.java b/src/main/java/com/conveyal/r5/analyst/TravelTimeReducer.java index 957b71646..990ad0aff 100644 --- a/src/main/java/com/conveyal/r5/analyst/TravelTimeReducer.java +++ b/src/main/java/com/conveyal/r5/analyst/TravelTimeReducer.java @@ -124,6 +124,8 @@ public TravelTimeReducer (AnalysisWorkerTask task, TransportNetwork network) { this.destinationPointSets = task.destinationPointSets; if (task instanceof TravelTimeSurfaceTask) { calculateTravelTimes = true; + // In single-point analyses, destination pointsets may be missing if the user has not selected one in the + // UI, or if the user has selected the step decay function instead of one of the other decay functions. calculateAccessibility = notNullOrEmpty(task.destinationPointSets); } else { // Maybe we should define recordAccessibility and recordTimes on the common superclass AnalysisWorkerTask. @@ -136,6 +138,8 @@ public TravelTimeReducer (AnalysisWorkerTask task, TransportNetwork network) { // These are conditionally instantiated because they can consume a lot of memory. if (calculateAccessibility) { accessibilityResult = new AccessibilityResult(task); + // TODO create this more selectively, choose histograms and nearest n independently + nearestNResult = new NearestNResult(task); } if (calculateTravelTimes) { travelTimeResult = new TravelTimeResult(task); @@ -143,9 +147,6 @@ public TravelTimeReducer (AnalysisWorkerTask task, TransportNetwork network) { if (task.includePathResults) { pathResult = new PathResult(task, network.transitLayer); } - if (task.includeNearestN) { - nearestNResult = new NearestNResult(task); - } // Validate and copy the travel time cutoffs, converting them to seconds to avoid repeated multiplication // in tight loops. Also find the points where the decay function reaches zero for these cutoffs. diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java index 69839d5ab..969ec3f9d 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java @@ -7,12 +7,14 @@ import com.conveyal.file.FileStorage; import com.conveyal.r5.OneOriginResult; import com.conveyal.r5.analyst.AccessibilityResult; +import com.conveyal.r5.analyst.NearestNResult.NearbyOpportunity; import com.conveyal.r5.analyst.NetworkPreloader; import com.conveyal.r5.analyst.PersistenceBuffer; import com.conveyal.r5.analyst.PointSetCache; import com.conveyal.r5.analyst.TravelTimeComputer; import com.conveyal.r5.analyst.error.TaskError; import com.conveyal.r5.common.JsonUtilities; +import com.conveyal.r5.transit.TransitLayer; import com.conveyal.r5.transit.TransportNetwork; import com.conveyal.r5.transit.TransportNetworkCache; import com.conveyal.r5.transitive.TransitiveNetwork; @@ -344,10 +346,10 @@ private byte[] singlePointResultToBinary ( timeGridWriter.writeToDataOutput(new LittleEndianDataOutputStream(byteArrayOutputStream)); addJsonToGrid( byteArrayOutputStream, - oneOriginResult.accessibility, + oneOriginResult, transportNetwork.scenarioApplicationWarnings, transportNetwork.scenarioApplicationInfo, - oneOriginResult.paths != null ? new PathResultSummary(oneOriginResult.paths, transportNetwork.transitLayer) : null + transportNetwork.transitLayer ); } // Single-point tasks don't have a job ID. For now, we'll categorize them by scenario ID. @@ -489,6 +491,10 @@ public static class GridJsonBlock { public PathResultSummary pathSummaries; + public NearbyOpportunity[][] nearby; + + public double[][][] opportunitiesPerMinute; + @Override public String toString () { return String.format( @@ -508,24 +514,34 @@ public String toString () { * and no serious errors, we use a success error code. * TODO distinguish between warnings and errors - we already distinguish between info and warnings. * This could be turned into a GridJsonBlock constructor, with the JSON writing code in an instance method. + * Note that this is very similar to the RegionalWorkResult constructor, and some duplication of logic could be + * achieved by somehow merging the two. */ public static void addJsonToGrid ( OutputStream outputStream, - AccessibilityResult accessibilityResult, + OneOriginResult oneOriginResult, List scenarioApplicationWarnings, List scenarioApplicationInfo, - PathResultSummary pathResult + TransitLayer transitLayer // Only used if oneOriginResult contains paths, can be null otherwise ) throws IOException { var jsonBlock = new GridJsonBlock(); jsonBlock.scenarioApplicationInfo = scenarioApplicationInfo; jsonBlock.scenarioApplicationWarnings = scenarioApplicationWarnings; - if (accessibilityResult != null) { - // Due to the application of distance decay functions, we may want to make the shift to non-integer - // accessibility values (especially for cases where there are relatively few opportunities across the whole - // study area). But we'd need to control the number of decimal places serialized into the JSON. - jsonBlock.accessibility = accessibilityResult.getIntValues(); + if (oneOriginResult != null) { + if (oneOriginResult.accessibility != null) { + // Due to the application of distance decay functions, we may want to make the shift to non-integer + // accessibility values (especially for cases where there are relatively few opportunities across the whole + // study area). But we'd need to control the number of decimal places serialized into the JSON. + jsonBlock.accessibility = oneOriginResult.accessibility.getIntValues(); + } + if (oneOriginResult.paths != null) { + jsonBlock.pathSummaries = new PathResultSummary(oneOriginResult.paths, transitLayer); + } + if (oneOriginResult.nearest != null) { + jsonBlock.nearby = oneOriginResult.nearest.nearby; + jsonBlock.opportunitiesPerMinute = oneOriginResult.nearest.opportunitiesPerMinute; + } } - jsonBlock.pathSummaries = pathResult; LOG.debug("Travel time surface written, appending {}.", jsonBlock); // We could do this when setting up the Spark handler, supplying writeValue as the response transformer // But then you also have to handle the case where you are returning raw bytes. diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorkerTask.java b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorkerTask.java index 15cd408ac..d1e88f78c 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorkerTask.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorkerTask.java @@ -99,9 +99,6 @@ public abstract class AnalysisWorkerTask extends ProfileRequest { */ public boolean includePathResults = false; - /** Whether to record the n closest destination points to the origin. */ - public boolean includeNearestN = false; - /** Whether to build a histogram of travel times to each destination, generally used in testing and debugging. */ public boolean recordTravelTimeHistograms = false; diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java b/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java index 0378012f7..4b6dd5d4e 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java @@ -42,7 +42,13 @@ public class RegionalWorkResult { * The nearest n destinations for each percentile of travel time. * Each item contains a travel time, target index, and ID. */ - public NearbyOpportunity[][] nearestN; + public NearbyOpportunity[][] nearby; + + /** + * The nearest n destinations for each percentile of travel time. + * Each item contains a travel time, target index, and ID. + */ + public double[][][] opportunitiesPerMinute; /** * If this field is non-null, the worker is reporting an error that compromises the quality of the result at this @@ -66,7 +72,8 @@ public RegionalWorkResult(OneOriginResult result, RegionalTask task) { this.travelTimeValues = result.travelTimes == null ? null : result.travelTimes.values; this.accessibilityValues = result.accessibility == null ? null : result.accessibility.getIntValues(); this.pathResult = result.paths == null ? null : result.paths.summarizeIterations(PathResult.Stat.MINIMUM); - this.nearestN = result.nearest == null ? null : result.nearest.opportunities; + this.nearby = result.nearest == null ? null : result.nearest.nearby; + this.opportunitiesPerMinute = result.nearest == null ? null : result.nearest.opportunitiesPerMinute; // TODO checkTravelTimeInvariants, checkAccessibilityInvariants to verify that values are monotonically increasing } diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/TravelTimeResult.java b/src/main/java/com/conveyal/r5/analyst/cluster/TravelTimeResult.java index c79f23ab8..2f484834c 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/TravelTimeResult.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/TravelTimeResult.java @@ -37,6 +37,9 @@ public class TravelTimeResult { * This is optional, generally used only for debugging and testing, so may be null if no histograms are recorded. * Since this is used for debugging and testing it might even be interesting to do it at one-second resolution or * just store every travel time, but binning makes it easy to compare with probability distributions. + * Note that this is one histogram _per target_ showing on how many iterations each travel time is the fastest, + * _not_ one histogram per origin/percentile showing how many destinations are reached at each travel time. The + * latter is essentially the discrete derivative of step-function accessibility and is tracked elsewhere (NearestNResult). */ int[][] histograms; From 655d0e2a616a19a070bbf93c93e55fe7342d2fb8 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Mon, 7 Aug 2023 18:19:09 +0800 Subject: [PATCH 3/9] provide dual accessibility, sorted opportunities - method to compute dual accessibility from temporal opportunity density - maintain nearby opportunities list in sorted order while constructing Still needs: - task parameters to enable these outputs - collation of results in regional analyses --- .../conveyal/r5/analyst/NearestNResult.java | 88 ++++++++++++++----- .../r5/analyst/cluster/AnalysisWorker.java | 3 + .../java/com/conveyal/r5/common/Util.java | 16 ++++ 3 files changed, 84 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/conveyal/r5/analyst/NearestNResult.java b/src/main/java/com/conveyal/r5/analyst/NearestNResult.java index 1df5a6b8d..0237ded76 100644 --- a/src/main/java/com/conveyal/r5/analyst/NearestNResult.java +++ b/src/main/java/com/conveyal/r5/analyst/NearestNResult.java @@ -3,13 +3,18 @@ import com.conveyal.r5.analyst.cluster.AnalysisWorkerTask; import com.google.common.base.Preconditions; +import static com.conveyal.r5.common.Util.newObjectArray; import static com.conveyal.r5.common.Util.notNullOrEmpty; /** - * An instance of this is included in a OneOriginResult for reporting the nearest N destinations. - * If we use more than one destination point set they must all be aligned with the same number of destinations. + * An instance of this is included in a OneOriginResult for reporting the nearest N destinations. If we use more than + * one destination point set they are already constrained to all be aligned with the same number of destinations. * - * This supports two kinds of results... (expand from comments on https://github.com/conveyal/r5/pull/884) + * The data retained here feed into three different kinds of results: "Dual" accessibility (the number of opportunities + * reached in a given number of minutes of travel time); temporal opportunity density (akin to a probability density + * function, how many opportunities are encountered during each minute of travel, whose integral is the cumulative + * accessibility curve); and the nearest one or more opportunities to a given origin. + * (expand from comments on https://github.com/conveyal/r5/pull/884) */ public class NearestNResult { @@ -18,20 +23,32 @@ public class NearestNResult { private final int nPercentiles; private final int nOpportunities; + /** + * Candidate for record instead of class with newer Java version. + */ public static class NearbyOpportunity { public int seconds = Integer.MAX_VALUE; public int target; public String id; + + @Override + public String toString() { + return "NearbyOpportunity{" + + "seconds=" + seconds + + ", target=" + target + + ", id='" + id + '\'' + + '}'; + } } // Fields to accumulate results - /** For each percentile, the closest N destinations */ + /** For each percentile, the closest N destinations with their IDs and how long it takes to reach them. */ public final NearbyOpportunity[][] nearby; /** - * For each destination set, for each percentile, for each minute of travel from 0 to 120, the number of - * opportunities reached in travel times from i (inclusive) to i+1 (exclusive). + * The temporal density of opportunities. For each destination set, for each percentile, for each minute of + * travel from 0 to 120, the number of opportunities reached in travel times from i (inclusive) to i+1 (exclusive). */ public final double[][][] opportunitiesPerMinute; @@ -47,8 +64,10 @@ public NearestNResult (AnalysisWorkerTask task) { this.opportunitiesPerMinute = new double[destinationPointSets.length][nPercentiles][120]; } + private int listLength = 0; // increment as lists grow in length; use as initial insert position + public void record (int target, int[] travelTimePercentilesSeconds) { - // Incremenet histogram bin for the number of minutes of travel, by the number of opportunities at the target. + // Increment histogram bin for the number of minutes of travel by the number of opportunities at the target. for (int d = 0; d < destinationPointSets.length; d++) { PointSet dps = destinationPointSets[d]; for (int p = 0; p < nPercentiles; p++) { @@ -66,27 +85,50 @@ public void record (int target, int[] travelTimePercentilesSeconds) { // selected any pointset in the UI (or if the step function is selected rather than another decay function). // So if any destinations are present, we only care about the first set. for (int p = 0; p < nPercentiles; p++) { - // Find the slot with the highest travel time less than the reported travel time for this percentile. - // TODO keep ordered by scanning from high end and shifting all elements to insert. - int toReplace = -1; - for (int i = 0; i < nOpportunities; i++) { - if (nearby[p][i] == null) { - nearby[p][i] = new NearbyOpportunity(); - toReplace = i; - break; + final int t = travelTimePercentilesSeconds[p]; // shorthand for current travel time + int i = listLength; // insertion point in list + // Shift elements to the right, dropping one off end of list, until insertion position is discovered. + while (i > 0 && nearby[p][i - 1] != null && t < nearby[p][i - 1].seconds) { + if (i < nOpportunities) { + nearby[p][i] = nearby[p][i - 1]; } - if (travelTimePercentilesSeconds[p] < nearby[p][i].seconds) { - if (toReplace < 0 || (nearby[p][i].seconds > nearby[p][toReplace].seconds)) { - toReplace = i; + i -= 1; + } + if (i < nOpportunities) { + final NearbyOpportunity tr = new NearbyOpportunity(); + tr.seconds = travelTimePercentilesSeconds[p]; + tr.target = target; + tr.id = destinationPointSets[0].getId(target); + nearby[p][i] = tr; + } + } + // If the lists in the arrays had not yet reached their full length, they will have grown by one. + if (listLength < nOpportunities) { + listLength += 1; + } + } + + /** + * Calculate "dual" accessibility from the accumulated temporal opportunity density array. + * @param n the threshold quantity of opportunities + * @return the number of minutes it takes to reach n opportunities, for each destination set and percentile of travel time. + */ + public int[][] minutesToReachOpporunities (int n) { + int[][] result = new int[destinationPointSets.length][nPercentiles]; + for (int d = 0; d < destinationPointSets.length; d++) { + for (int p = 0; p < nPercentiles; p++) { + result[d][p] = -1; + double count = 0; + for (int m = 0; m < 120; m++) { + count += opportunitiesPerMinute[d][p][m]; + if (count >= n) { + result[d][p] = m + 1; + break; } } } - if (toReplace >= 0) { - nearby[p][toReplace].seconds = travelTimePercentilesSeconds[p]; - nearby[p][toReplace].target = target; - nearby[p][toReplace].id = destinationPointSets[0].getId(target); - } } + return result; } } diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java index 969ec3f9d..5be3ddcf8 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java @@ -540,6 +540,9 @@ public static void addJsonToGrid ( if (oneOriginResult.nearest != null) { jsonBlock.nearby = oneOriginResult.nearest.nearby; jsonBlock.opportunitiesPerMinute = oneOriginResult.nearest.opportunitiesPerMinute; + LOG.info("Dual accessibility: {}", oneOriginResult.nearest.minutesToReachOpporunities(272_500)); + LOG.info("Opportunities per minute: {}", oneOriginResult.nearest.opportunitiesPerMinute); + LOG.info("Opportunities nearby: {}", oneOriginResult.nearest.nearby); } } LOG.debug("Travel time surface written, appending {}.", jsonBlock); diff --git a/src/main/java/com/conveyal/r5/common/Util.java b/src/main/java/com/conveyal/r5/common/Util.java index ddfce26d5..c79e1c168 100644 --- a/src/main/java/com/conveyal/r5/common/Util.java +++ b/src/main/java/com/conveyal/r5/common/Util.java @@ -3,6 +3,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Map; +import java.util.function.Supplier; public abstract class Util { @@ -66,4 +67,19 @@ public static int[] newIntArray (int length, int defaultValue) { return array; } + /** + * Convenience method to create a 2D array and fill it immediately with new instances of a class. + * The supplier can be a method reference to a constructor like ToBeInstantiated::new, and the returned + * array will be of that type. + */ + public static T[][] newObjectArray (int d1, int d2, Supplier supplier) { + T[][] result = (T[][]) new Object[d1][d2]; + for (int x = 0; x < d1; x++) { + for (int y = 0; y < d2; y++) { + result[x][y] = supplier.get(); + } + } + return result; + } + } From 6a03c8aa147fd9bad23bb848983248c1e2a06185 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 8 Aug 2023 21:06:50 +0800 Subject: [PATCH 4/9] allow enabling and recording new results - Enable opportunity temporal density, nearest n opportunities, and dual accessibility in AnalysisRequest and AnalysisWorkerTask - CSVResultWriter recording opportunity density and dual accessibility --- .../analysis/models/AnalysisRequest.java | 13 +++ .../analysis/results/CsvResultType.java | 2 +- .../results/MultiOriginAssembler.java | 4 + .../results/OpportunityCsvResultWriter.java | 90 +++++++++++++++++++ .../conveyal/r5/analyst/NearestNResult.java | 18 ++-- .../r5/analyst/TravelTimeReducer.java | 7 +- .../r5/analyst/cluster/AnalysisWorker.java | 1 - .../analyst/cluster/AnalysisWorkerTask.java | 9 ++ .../analyst/cluster/RegionalWorkResult.java | 3 +- 9 files changed, 134 insertions(+), 13 deletions(-) create mode 100644 src/main/java/com/conveyal/analysis/results/OpportunityCsvResultWriter.java diff --git a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java index 10d326656..82f0676a6 100644 --- a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java +++ b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java @@ -161,6 +161,16 @@ public class AnalysisRequest { */ public ChaosParameters injectFault; + /** + * Whether to include the number of opportunities reached during each minute of travel in results sent back + * to the broker. Requires both an origin and destination pointset to be specified, and in the case of regional + * analyses the origins must be non-gridded. (Should be possible to make a grid as well.) + */ + public boolean opportunityTemporalDensity = false; + + public int dualAccessibilityOpportunityThreshold = 0; + + /** * Create the R5 `Scenario` from this request. */ @@ -265,6 +275,9 @@ public void populateTask (AnalysisWorkerTask task, UserPermissions userPermissio throw new IllegalArgumentException("Must be admin user to inject faults."); } } + + task.opportunityTemporalDensity = opportunityTemporalDensity; + task.dualAccessibilityOpportunityThreshold = dualAccessibilityOpportunityThreshold; } private EnumSet getEnumSetFromString (String s) { diff --git a/src/main/java/com/conveyal/analysis/results/CsvResultType.java b/src/main/java/com/conveyal/analysis/results/CsvResultType.java index cdf60e410..56612aafc 100644 --- a/src/main/java/com/conveyal/analysis/results/CsvResultType.java +++ b/src/main/java/com/conveyal/analysis/results/CsvResultType.java @@ -5,5 +5,5 @@ * do serve to enumerate the acceptable parameters coming over the HTTP API. */ public enum CsvResultType { - ACCESS, TIMES, PATHS + ACCESS, TIMES, PATHS, OPPORTUNITIES } diff --git a/src/main/java/com/conveyal/analysis/results/MultiOriginAssembler.java b/src/main/java/com/conveyal/analysis/results/MultiOriginAssembler.java index 3bbd5915f..9dcce90e3 100644 --- a/src/main/java/com/conveyal/analysis/results/MultiOriginAssembler.java +++ b/src/main/java/com/conveyal/analysis/results/MultiOriginAssembler.java @@ -123,6 +123,10 @@ public MultiOriginAssembler (RegionalAnalysis regionalAnalysis, Job job, FileSto resultWriters.add(new PathCsvResultWriter(job.templateTask, fileStorage)); } + if (job.templateTask.opportunityTemporalDensity) { + resultWriters.add(new OpportunityCsvResultWriter(job.templateTask, fileStorage)); + } + checkArgument(job.templateTask.makeTauiSite || notNullOrEmpty(resultWriters), "A non-Taui regional analysis should always create at least one grid or CSV file."); diff --git a/src/main/java/com/conveyal/analysis/results/OpportunityCsvResultWriter.java b/src/main/java/com/conveyal/analysis/results/OpportunityCsvResultWriter.java new file mode 100644 index 000000000..dfe741602 --- /dev/null +++ b/src/main/java/com/conveyal/analysis/results/OpportunityCsvResultWriter.java @@ -0,0 +1,90 @@ +package com.conveyal.analysis.results; + +import com.conveyal.file.FileStorage; +import com.conveyal.r5.analyst.cluster.RegionalTask; +import com.conveyal.r5.analyst.cluster.RegionalWorkResult; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * This handles collating regional results into CSV files containing temporal opportunity density + * (number of opportunities reached in each one-minute interval, the derivative of step-function accessibility) + * as well as "dual" accessibility (the amount of time needed to reach n opportunities). + * And maybe the N closest opportunities to each origin? + */ +public class OpportunityCsvResultWriter extends CsvResultWriter { + + private final int dualOpportunityCount; + + public OpportunityCsvResultWriter(RegionalTask task, FileStorage fileStorage) throws IOException { + super(task, fileStorage); + dualOpportunityCount = task.dualAccessibilityOpportunityThreshold; + } + + @Override + public CsvResultType resultType () { + return CsvResultType.OPPORTUNITIES; + } + + @Override + public String[] columnHeaders () { + List headers = new ArrayList<>(); + // The ids of the freeform origin point and destination set + headers.add("originId"); + headers.add("destId"); + headers.add("percentile"); + for (int m = 0; m < 120; m += 1) { + // The opportunity density over travel minute m + headers.add(Integer.toString(m)); + } + // The number of minutes needed to reach d destination opportunities + headers.add("D" + dualOpportunityCount); + return headers.toArray(new String[0]); + } + + @Override + protected void checkDimension (RegionalWorkResult workResult) { + checkDimension(workResult, "destination pointsets", workResult.opportunitiesPerMinute.length, task.destinationPointSetKeys.length); + for (double[][] percentilesForPointset : workResult.opportunitiesPerMinute) { + checkDimension(workResult, "percentiles", percentilesForPointset.length, task.percentiles.length); + for (double[] minutesForPercentile : percentilesForPointset) { + checkDimension(workResult, "minutes", minutesForPercentile.length, 120); + } + } + } + + @Override + public Iterable rowValues (RegionalWorkResult workResult) { + List row = new ArrayList<>(125); + String originId = task.originPointSet.getId(workResult.taskId); + for (int d = 0; d < task.destinationPointSetKeys.length; d++) { + int[][] percentilesForDestPointset = workResult.accessibilityValues[d]; + for (int p = 0; p < task.percentiles.length; p++) { + row.add(originId); + row.add(task.destinationPointSets[d].name); + row.add(Integer.toString(p)); + int[] densitiesPerMinute = percentilesForDestPointset[p]; + for (int m = 0; m < 120; m++) { + row.add(Double.toString(densitiesPerMinute[m])); + } + // Dual accessibility + { + int m = 0; + double sum = 0; + while (sum < dualOpportunityCount) { + sum += densitiesPerMinute[m]; + m += 1; + } + row.add(Integer.toString(m >= 120 ? -1 : m)); + } + } + } + // List.of() or Arrays.asList() don't work without explicitly specifying the generic type because + // they interpret the String[] as varargs in the method signature. + return List.of(row.toArray(new String[0])); + } + +} diff --git a/src/main/java/com/conveyal/r5/analyst/NearestNResult.java b/src/main/java/com/conveyal/r5/analyst/NearestNResult.java index 0237ded76..7b02ffef7 100644 --- a/src/main/java/com/conveyal/r5/analyst/NearestNResult.java +++ b/src/main/java/com/conveyal/r5/analyst/NearestNResult.java @@ -3,8 +3,8 @@ import com.conveyal.r5.analyst.cluster.AnalysisWorkerTask; import com.google.common.base.Preconditions; -import static com.conveyal.r5.common.Util.newObjectArray; import static com.conveyal.r5.common.Util.notNullOrEmpty; +import static com.conveyal.r5.profile.FastRaptorWorker.UNREACHED; /** * An instance of this is included in a OneOriginResult for reporting the nearest N destinations. If we use more than @@ -15,6 +15,9 @@ * function, how many opportunities are encountered during each minute of travel, whose integral is the cumulative * accessibility curve); and the nearest one or more opportunities to a given origin. * (expand from comments on https://github.com/conveyal/r5/pull/884) + * + * Corresponds to OpportunityCsvResultWriter when collating regional results, and + * AnalysisWorkerTask#opportunityTemporalDensity to enable, so maybe the names should be made more coherent. */ public class NearestNResult { @@ -66,14 +69,17 @@ public NearestNResult (AnalysisWorkerTask task) { private int listLength = 0; // increment as lists grow in length; use as initial insert position - public void record (int target, int[] travelTimePercentilesSeconds) { + public void recordOneTarget (int target, int[] travelTimePercentilesSeconds) { // Increment histogram bin for the number of minutes of travel by the number of opportunities at the target. for (int d = 0; d < destinationPointSets.length; d++) { PointSet dps = destinationPointSets[d]; for (int p = 0; p < nPercentiles; p++) { - int i = travelTimePercentilesSeconds[p] / 60; - if (i <= 120) { - opportunitiesPerMinute[d][p][i] += dps.getOpportunityCount(target); + if (travelTimePercentilesSeconds[p] == UNREACHED) { + break; // If any percentile is unreached, all higher ones are also unreached. + } + int m = travelTimePercentilesSeconds[p] / 60; + if (m <= 120) { + opportunitiesPerMinute[d][p][m] += dps.getOpportunityCount(target); } } } @@ -113,7 +119,7 @@ public void record (int target, int[] travelTimePercentilesSeconds) { * @param n the threshold quantity of opportunities * @return the number of minutes it takes to reach n opportunities, for each destination set and percentile of travel time. */ - public int[][] minutesToReachOpporunities (int n) { + public int[][] minutesToReachOpportunities(int n) { int[][] result = new int[destinationPointSets.length][nPercentiles]; for (int d = 0; d < destinationPointSets.length; d++) { for (int p = 0; p < nPercentiles; p++) { diff --git a/src/main/java/com/conveyal/r5/analyst/TravelTimeReducer.java b/src/main/java/com/conveyal/r5/analyst/TravelTimeReducer.java index 990ad0aff..5cb5b5185 100644 --- a/src/main/java/com/conveyal/r5/analyst/TravelTimeReducer.java +++ b/src/main/java/com/conveyal/r5/analyst/TravelTimeReducer.java @@ -138,8 +138,6 @@ public TravelTimeReducer (AnalysisWorkerTask task, TransportNetwork network) { // These are conditionally instantiated because they can consume a lot of memory. if (calculateAccessibility) { accessibilityResult = new AccessibilityResult(task); - // TODO create this more selectively, choose histograms and nearest n independently - nearestNResult = new NearestNResult(task); } if (calculateTravelTimes) { travelTimeResult = new TravelTimeResult(task); @@ -147,6 +145,9 @@ public TravelTimeReducer (AnalysisWorkerTask task, TransportNetwork network) { if (task.includePathResults) { pathResult = new PathResult(task, network.transitLayer); } + if (task.opportunityTemporalDensity) { + nearestNResult = new NearestNResult(task); + } // Validate and copy the travel time cutoffs, converting them to seconds to avoid repeated multiplication // in tight loops. Also find the points where the decay function reaches zero for these cutoffs. @@ -279,7 +280,7 @@ private void recordTravelTimePercentilesForTarget (int target, int[] travelTimeP } } if (nearestNResult != null) { - nearestNResult.record(target, travelTimePercentilesSeconds); + nearestNResult.recordOneTarget(target, travelTimePercentilesSeconds); } } diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java index 5be3ddcf8..7d951a9a0 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java @@ -540,7 +540,6 @@ public static void addJsonToGrid ( if (oneOriginResult.nearest != null) { jsonBlock.nearby = oneOriginResult.nearest.nearby; jsonBlock.opportunitiesPerMinute = oneOriginResult.nearest.opportunitiesPerMinute; - LOG.info("Dual accessibility: {}", oneOriginResult.nearest.minutesToReachOpporunities(272_500)); LOG.info("Opportunities per minute: {}", oneOriginResult.nearest.opportunitiesPerMinute); LOG.info("Opportunities nearby: {}", oneOriginResult.nearest.nearby); } diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorkerTask.java b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorkerTask.java index d1e88f78c..547b4123f 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorkerTask.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorkerTask.java @@ -99,6 +99,15 @@ public abstract class AnalysisWorkerTask extends ProfileRequest { */ public boolean includePathResults = false; + /** + * Whether to include the number of opportunities reached during each minute of travel in results sent back + * to the broker. Requires both an origin and destination pointset to be specified, and in the case of regional + * analyses the origins must be non-gridded. (Should be possible to make a grid as well.) + */ + public boolean opportunityTemporalDensity = false; + + public int dualAccessibilityOpportunityThreshold = 0; + /** Whether to build a histogram of travel times to each destination, generally used in testing and debugging. */ public boolean recordTravelTimeHistograms = false; diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java b/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java index 4b6dd5d4e..5fa619c1a 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java @@ -45,8 +45,7 @@ public class RegionalWorkResult { public NearbyOpportunity[][] nearby; /** - * The nearest n destinations for each percentile of travel time. - * Each item contains a travel time, target index, and ID. + * The temporal density of opportunities - how many are reached during each minute of travel. */ public double[][][] opportunitiesPerMinute; From 4051ddb2443b04e04e0ed7aa7fed73b734bf130c Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Thu, 17 Aug 2023 14:44:20 +0800 Subject: [PATCH 5/9] remove tracking of closest N opportunities Simpler and more maintainable, given that there's no immediate demand for this information among users. --- .../conveyal/r5/analyst/NearestNResult.java | 66 +++---------------- .../r5/analyst/cluster/AnalysisWorker.java | 5 -- .../analyst/cluster/RegionalWorkResult.java | 9 +-- 3 files changed, 10 insertions(+), 70 deletions(-) diff --git a/src/main/java/com/conveyal/r5/analyst/NearestNResult.java b/src/main/java/com/conveyal/r5/analyst/NearestNResult.java index 7b02ffef7..4e6de22b3 100644 --- a/src/main/java/com/conveyal/r5/analyst/NearestNResult.java +++ b/src/main/java/com/conveyal/r5/analyst/NearestNResult.java @@ -16,42 +16,27 @@ * accessibility curve); and the nearest one or more opportunities to a given origin. * (expand from comments on https://github.com/conveyal/r5/pull/884) * + * Originally this class was tracking the identity of the N nearest points rather than just binning them by travel time. + * This is more efficient in cases where N is small, and allows retaining the one-second resolution. However currently + * there does not seem to be much demand among users for this level of detail, so it has been removed in the interest + * of simplicity and maintainability. + * * Corresponds to OpportunityCsvResultWriter when collating regional results, and * AnalysisWorkerTask#opportunityTemporalDensity to enable, so maybe the names should be made more coherent. */ public class NearestNResult { - private static final int DEFAULT_N_OPPORTUNITIES = 3; + // Internal state fields + private final PointSet[] destinationPointSets; private final int nPercentiles; - private final int nOpportunities; - - /** - * Candidate for record instead of class with newer Java version. - */ - public static class NearbyOpportunity { - public int seconds = Integer.MAX_VALUE; - public int target; - public String id; - - @Override - public String toString() { - return "NearbyOpportunity{" + - "seconds=" + seconds + - ", target=" + target + - ", id='" + id + '\'' + - '}'; - } - } - // Fields to accumulate results - - /** For each percentile, the closest N destinations with their IDs and how long it takes to reach them. */ - public final NearbyOpportunity[][] nearby; + // Externally visible fields for accumulating results /** * The temporal density of opportunities. For each destination set, for each percentile, for each minute of * travel from 0 to 120, the number of opportunities reached in travel times from i (inclusive) to i+1 (exclusive). + * */ public final double[][][] opportunitiesPerMinute; @@ -62,13 +47,9 @@ public NearestNResult (AnalysisWorkerTask task) { ); this.destinationPointSets = task.destinationPointSets; this.nPercentiles = task.percentiles.length; - this.nOpportunities = DEFAULT_N_OPPORTUNITIES; - this.nearby = new NearbyOpportunity[nPercentiles][nOpportunities]; this.opportunitiesPerMinute = new double[destinationPointSets.length][nPercentiles][120]; } - private int listLength = 0; // increment as lists grow in length; use as initial insert position - public void recordOneTarget (int target, int[] travelTimePercentilesSeconds) { // Increment histogram bin for the number of minutes of travel by the number of opportunities at the target. for (int d = 0; d < destinationPointSets.length; d++) { @@ -83,35 +64,6 @@ public void recordOneTarget (int target, int[] travelTimePercentilesSeconds) { } } } - // Insert the destination in the list of nearest destinations if it improves on those already present. - // Nearest N only makes sense for freeform point sets that define IDs for each point. If there were - // multiple destination point sets, each of them could define a different ID for the same point. - // However unlike gridded pointsets, we don't allow more than one freeform pointset in the same request. - // Note also that in single-point analyses, the destination pointsets may be null if the user has not - // selected any pointset in the UI (or if the step function is selected rather than another decay function). - // So if any destinations are present, we only care about the first set. - for (int p = 0; p < nPercentiles; p++) { - final int t = travelTimePercentilesSeconds[p]; // shorthand for current travel time - int i = listLength; // insertion point in list - // Shift elements to the right, dropping one off end of list, until insertion position is discovered. - while (i > 0 && nearby[p][i - 1] != null && t < nearby[p][i - 1].seconds) { - if (i < nOpportunities) { - nearby[p][i] = nearby[p][i - 1]; - } - i -= 1; - } - if (i < nOpportunities) { - final NearbyOpportunity tr = new NearbyOpportunity(); - tr.seconds = travelTimePercentilesSeconds[p]; - tr.target = target; - tr.id = destinationPointSets[0].getId(target); - nearby[p][i] = tr; - } - } - // If the lists in the arrays had not yet reached their full length, they will have grown by one. - if (listLength < nOpportunities) { - listLength += 1; - } } /** diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java index 7d951a9a0..aa16d6838 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java @@ -7,7 +7,6 @@ import com.conveyal.file.FileStorage; import com.conveyal.r5.OneOriginResult; import com.conveyal.r5.analyst.AccessibilityResult; -import com.conveyal.r5.analyst.NearestNResult.NearbyOpportunity; import com.conveyal.r5.analyst.NetworkPreloader; import com.conveyal.r5.analyst.PersistenceBuffer; import com.conveyal.r5.analyst.PointSetCache; @@ -491,8 +490,6 @@ public static class GridJsonBlock { public PathResultSummary pathSummaries; - public NearbyOpportunity[][] nearby; - public double[][][] opportunitiesPerMinute; @Override @@ -538,10 +535,8 @@ public static void addJsonToGrid ( jsonBlock.pathSummaries = new PathResultSummary(oneOriginResult.paths, transitLayer); } if (oneOriginResult.nearest != null) { - jsonBlock.nearby = oneOriginResult.nearest.nearby; jsonBlock.opportunitiesPerMinute = oneOriginResult.nearest.opportunitiesPerMinute; LOG.info("Opportunities per minute: {}", oneOriginResult.nearest.opportunitiesPerMinute); - LOG.info("Opportunities nearby: {}", oneOriginResult.nearest.nearby); } } LOG.debug("Travel time surface written, appending {}.", jsonBlock); diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java b/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java index 5fa619c1a..389902c8b 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java @@ -1,7 +1,6 @@ package com.conveyal.r5.analyst.cluster; import com.conveyal.r5.OneOriginResult; -import com.conveyal.r5.analyst.NearestNResult.NearbyOpportunity; import com.conveyal.r5.util.ExceptionUtils; import java.util.ArrayList; @@ -38,14 +37,9 @@ public class RegionalWorkResult { */ public int[][][] accessibilityValues; - /** - * The nearest n destinations for each percentile of travel time. - * Each item contains a travel time, target index, and ID. - */ - public NearbyOpportunity[][] nearby; - /** * The temporal density of opportunities - how many are reached during each minute of travel. + * Quantities of opportunities for each [destinationGrid, percentile, minute]. */ public double[][][] opportunitiesPerMinute; @@ -71,7 +65,6 @@ public RegionalWorkResult(OneOriginResult result, RegionalTask task) { this.travelTimeValues = result.travelTimes == null ? null : result.travelTimes.values; this.accessibilityValues = result.accessibility == null ? null : result.accessibility.getIntValues(); this.pathResult = result.paths == null ? null : result.paths.summarizeIterations(PathResult.Stat.MINIMUM); - this.nearby = result.nearest == null ? null : result.nearest.nearby; this.opportunitiesPerMinute = result.nearest == null ? null : result.nearest.opportunitiesPerMinute; // TODO checkTravelTimeInvariants, checkAccessibilityInvariants to verify that values are monotonically increasing } From e8944efb07926cf6a54d1f630222c565d559f650 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Thu, 17 Aug 2023 15:18:25 +0800 Subject: [PATCH 6/9] make class and field naming more consistent --- .../analysis/models/AnalysisRequest.java | 4 ++-- .../results/MultiOriginAssembler.java | 4 ++-- ...va => TemporalDensityCsvResultWriter.java} | 19 ++++++++++--------- .../java/com/conveyal/r5/OneOriginResult.java | 8 ++++---- ...Result.java => TemporalDensityResult.java} | 7 ++----- .../r5/analyst/TravelTimeReducer.java | 14 +++++++------- .../r5/analyst/cluster/AnalysisWorker.java | 6 +++--- .../analyst/cluster/AnalysisWorkerTask.java | 11 ++++++++--- .../analyst/cluster/RegionalWorkResult.java | 2 +- .../r5/analyst/cluster/TravelTimeResult.java | 2 +- 10 files changed, 40 insertions(+), 37 deletions(-) rename src/main/java/com/conveyal/analysis/results/{OpportunityCsvResultWriter.java => TemporalDensityCsvResultWriter.java} (83%) rename src/main/java/com/conveyal/r5/analyst/{NearestNResult.java => TemporalDensityResult.java} (93%) diff --git a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java index 82f0676a6..1a62a21ff 100644 --- a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java +++ b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java @@ -276,8 +276,8 @@ public void populateTask (AnalysisWorkerTask task, UserPermissions userPermissio } } - task.opportunityTemporalDensity = opportunityTemporalDensity; - task.dualAccessibilityOpportunityThreshold = dualAccessibilityOpportunityThreshold; + task.includeTemporalDensity = opportunityTemporalDensity; + task.dualAccessibilityThreshold = dualAccessibilityOpportunityThreshold; } private EnumSet getEnumSetFromString (String s) { diff --git a/src/main/java/com/conveyal/analysis/results/MultiOriginAssembler.java b/src/main/java/com/conveyal/analysis/results/MultiOriginAssembler.java index 9dcce90e3..d5c5094b2 100644 --- a/src/main/java/com/conveyal/analysis/results/MultiOriginAssembler.java +++ b/src/main/java/com/conveyal/analysis/results/MultiOriginAssembler.java @@ -123,8 +123,8 @@ public MultiOriginAssembler (RegionalAnalysis regionalAnalysis, Job job, FileSto resultWriters.add(new PathCsvResultWriter(job.templateTask, fileStorage)); } - if (job.templateTask.opportunityTemporalDensity) { - resultWriters.add(new OpportunityCsvResultWriter(job.templateTask, fileStorage)); + if (job.templateTask.includeTemporalDensity) { + resultWriters.add(new TemporalDensityCsvResultWriter(job.templateTask, fileStorage)); } checkArgument(job.templateTask.makeTauiSite || notNullOrEmpty(resultWriters), diff --git a/src/main/java/com/conveyal/analysis/results/OpportunityCsvResultWriter.java b/src/main/java/com/conveyal/analysis/results/TemporalDensityCsvResultWriter.java similarity index 83% rename from src/main/java/com/conveyal/analysis/results/OpportunityCsvResultWriter.java rename to src/main/java/com/conveyal/analysis/results/TemporalDensityCsvResultWriter.java index dfe741602..f9ed4472b 100644 --- a/src/main/java/com/conveyal/analysis/results/OpportunityCsvResultWriter.java +++ b/src/main/java/com/conveyal/analysis/results/TemporalDensityCsvResultWriter.java @@ -6,22 +6,20 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; /** * This handles collating regional results into CSV files containing temporal opportunity density * (number of opportunities reached in each one-minute interval, the derivative of step-function accessibility) * as well as "dual" accessibility (the amount of time needed to reach n opportunities). - * And maybe the N closest opportunities to each origin? */ -public class OpportunityCsvResultWriter extends CsvResultWriter { +public class TemporalDensityCsvResultWriter extends CsvResultWriter { - private final int dualOpportunityCount; + private final int dualThreshold; - public OpportunityCsvResultWriter(RegionalTask task, FileStorage fileStorage) throws IOException { + public TemporalDensityCsvResultWriter(RegionalTask task, FileStorage fileStorage) throws IOException { super(task, fileStorage); - dualOpportunityCount = task.dualAccessibilityOpportunityThreshold; + dualThreshold = task.dualAccessibilityThreshold; } @Override @@ -41,13 +39,16 @@ public String[] columnHeaders () { headers.add(Integer.toString(m)); } // The number of minutes needed to reach d destination opportunities - headers.add("D" + dualOpportunityCount); + headers.add("D" + dualThreshold); return headers.toArray(new String[0]); } @Override protected void checkDimension (RegionalWorkResult workResult) { - checkDimension(workResult, "destination pointsets", workResult.opportunitiesPerMinute.length, task.destinationPointSetKeys.length); + checkDimension( + workResult, "destination pointsets", + workResult.opportunitiesPerMinute.length, task.destinationPointSetKeys.length + ); for (double[][] percentilesForPointset : workResult.opportunitiesPerMinute) { checkDimension(workResult, "percentiles", percentilesForPointset.length, task.percentiles.length); for (double[] minutesForPercentile : percentilesForPointset) { @@ -74,7 +75,7 @@ public Iterable rowValues (RegionalWorkResult workResult) { { int m = 0; double sum = 0; - while (sum < dualOpportunityCount) { + while (sum < dualThreshold) { sum += densitiesPerMinute[m]; m += 1; } diff --git a/src/main/java/com/conveyal/r5/OneOriginResult.java b/src/main/java/com/conveyal/r5/OneOriginResult.java index 24e9348b4..090cdd9a0 100644 --- a/src/main/java/com/conveyal/r5/OneOriginResult.java +++ b/src/main/java/com/conveyal/r5/OneOriginResult.java @@ -1,7 +1,7 @@ package com.conveyal.r5; import com.conveyal.r5.analyst.AccessibilityResult; -import com.conveyal.r5.analyst.NearestNResult; +import com.conveyal.r5.analyst.TemporalDensityResult; import com.conveyal.r5.analyst.cluster.PathResult; import com.conveyal.r5.analyst.cluster.TravelTimeResult; @@ -21,18 +21,18 @@ public class OneOriginResult { public final PathResult paths; - public final NearestNResult nearest; + public final TemporalDensityResult density; public OneOriginResult( TravelTimeResult travelTimes, AccessibilityResult accessibility, PathResult paths, - NearestNResult nearest + TemporalDensityResult density ) { this.travelTimes = travelTimes; this.accessibility = accessibility; this.paths = paths; - this.nearest = nearest; + this.density = density; } } diff --git a/src/main/java/com/conveyal/r5/analyst/NearestNResult.java b/src/main/java/com/conveyal/r5/analyst/TemporalDensityResult.java similarity index 93% rename from src/main/java/com/conveyal/r5/analyst/NearestNResult.java rename to src/main/java/com/conveyal/r5/analyst/TemporalDensityResult.java index 4e6de22b3..5a4723204 100644 --- a/src/main/java/com/conveyal/r5/analyst/NearestNResult.java +++ b/src/main/java/com/conveyal/r5/analyst/TemporalDensityResult.java @@ -20,11 +20,8 @@ * This is more efficient in cases where N is small, and allows retaining the one-second resolution. However currently * there does not seem to be much demand among users for this level of detail, so it has been removed in the interest * of simplicity and maintainability. - * - * Corresponds to OpportunityCsvResultWriter when collating regional results, and - * AnalysisWorkerTask#opportunityTemporalDensity to enable, so maybe the names should be made more coherent. */ -public class NearestNResult { +public class TemporalDensityResult { // Internal state fields @@ -40,7 +37,7 @@ public class NearestNResult { */ public final double[][][] opportunitiesPerMinute; - public NearestNResult (AnalysisWorkerTask task) { + public TemporalDensityResult(AnalysisWorkerTask task) { Preconditions.checkArgument( notNullOrEmpty(task.destinationPointSets), "Dual accessibility requires at least one destination pointset." diff --git a/src/main/java/com/conveyal/r5/analyst/TravelTimeReducer.java b/src/main/java/com/conveyal/r5/analyst/TravelTimeReducer.java index 5cb5b5185..15ce02012 100644 --- a/src/main/java/com/conveyal/r5/analyst/TravelTimeReducer.java +++ b/src/main/java/com/conveyal/r5/analyst/TravelTimeReducer.java @@ -46,8 +46,8 @@ public class TravelTimeReducer { /** Retains the paths to one or all destinations, for recording in CSV or reporting in the UI. */ private PathResult pathResult = null; - /** Retains the nearest N destinations from this origin. */ - private NearestNResult nearestNResult = null; + /** Represents how many destinations are reached in each minute of travel from this origin. */ + private TemporalDensityResult temporalDensityResult = null; /** If we are calculating accessibility, the PointSets containing opportunities. */ private PointSet[] destinationPointSets; @@ -145,8 +145,8 @@ public TravelTimeReducer (AnalysisWorkerTask task, TransportNetwork network) { if (task.includePathResults) { pathResult = new PathResult(task, network.transitLayer); } - if (task.opportunityTemporalDensity) { - nearestNResult = new NearestNResult(task); + if (task.includeTemporalDensity) { + temporalDensityResult = new TemporalDensityResult(task); } // Validate and copy the travel time cutoffs, converting them to seconds to avoid repeated multiplication @@ -279,8 +279,8 @@ private void recordTravelTimePercentilesForTarget (int target, int[] travelTimeP } } } - if (nearestNResult != null) { - nearestNResult.recordOneTarget(target, travelTimePercentilesSeconds); + if (temporalDensityResult != null) { + temporalDensityResult.recordOneTarget(target, travelTimePercentilesSeconds); } } @@ -340,7 +340,7 @@ private int convertToMinutes (int timeSeconds) { * origin point is not connected to the street network. */ public OneOriginResult finish () { - return new OneOriginResult(travelTimeResult, accessibilityResult, pathResult, nearestNResult); + return new OneOriginResult(travelTimeResult, accessibilityResult, pathResult, temporalDensityResult); } /** diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java index aa16d6838..2104d16ee 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java @@ -534,9 +534,9 @@ public static void addJsonToGrid ( if (oneOriginResult.paths != null) { jsonBlock.pathSummaries = new PathResultSummary(oneOriginResult.paths, transitLayer); } - if (oneOriginResult.nearest != null) { - jsonBlock.opportunitiesPerMinute = oneOriginResult.nearest.opportunitiesPerMinute; - LOG.info("Opportunities per minute: {}", oneOriginResult.nearest.opportunitiesPerMinute); + if (oneOriginResult.density != null) { + jsonBlock.opportunitiesPerMinute = oneOriginResult.density.opportunitiesPerMinute; + LOG.info("Opportunities per minute: {}", oneOriginResult.density.opportunitiesPerMinute); } } LOG.debug("Travel time surface written, appending {}.", jsonBlock); diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorkerTask.java b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorkerTask.java index 547b4123f..f3305e280 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorkerTask.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorkerTask.java @@ -102,11 +102,16 @@ public abstract class AnalysisWorkerTask extends ProfileRequest { /** * Whether to include the number of opportunities reached during each minute of travel in results sent back * to the broker. Requires both an origin and destination pointset to be specified, and in the case of regional - * analyses the origins must be non-gridded. (Should be possible to make a grid as well.) + * analyses the origins must be non-gridded, and results will be collated to CSV. + * It should be possible to enable regional results for gridded origins as well. */ - public boolean opportunityTemporalDensity = false; + public boolean includeTemporalDensity = false; - public int dualAccessibilityOpportunityThreshold = 0; + /** + * If this is set to a value above zero, report the amount of time needed to reach the given number of + * opportunities from this origin (known technically as "dual accessibility"). + */ + public int dualAccessibilityThreshold = 0; /** Whether to build a histogram of travel times to each destination, generally used in testing and debugging. */ public boolean recordTravelTimeHistograms = false; diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java b/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java index 389902c8b..d15531c03 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java @@ -65,7 +65,7 @@ public RegionalWorkResult(OneOriginResult result, RegionalTask task) { this.travelTimeValues = result.travelTimes == null ? null : result.travelTimes.values; this.accessibilityValues = result.accessibility == null ? null : result.accessibility.getIntValues(); this.pathResult = result.paths == null ? null : result.paths.summarizeIterations(PathResult.Stat.MINIMUM); - this.opportunitiesPerMinute = result.nearest == null ? null : result.nearest.opportunitiesPerMinute; + this.opportunitiesPerMinute = result.density == null ? null : result.density.opportunitiesPerMinute; // TODO checkTravelTimeInvariants, checkAccessibilityInvariants to verify that values are monotonically increasing } diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/TravelTimeResult.java b/src/main/java/com/conveyal/r5/analyst/cluster/TravelTimeResult.java index 2f484834c..c851afcac 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/TravelTimeResult.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/TravelTimeResult.java @@ -39,7 +39,7 @@ public class TravelTimeResult { * just store every travel time, but binning makes it easy to compare with probability distributions. * Note that this is one histogram _per target_ showing on how many iterations each travel time is the fastest, * _not_ one histogram per origin/percentile showing how many destinations are reached at each travel time. The - * latter is essentially the discrete derivative of step-function accessibility and is tracked elsewhere (NearestNResult). + * latter is essentially the discrete derivative of step-function accessibility and is tracked elsewhere (TemporalDensityResult). */ int[][] histograms; From e68836570b7c323deb07ad3b6d1a18540f03d4b7 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Thu, 17 Aug 2023 17:52:01 +0800 Subject: [PATCH 7/9] consistent naming, comments, fix csv writer --- .../analysis/models/AnalysisRequest.java | 15 ++++++--- .../analysis/results/CsvResultType.java | 2 +- .../results/MultiOriginAssembler.java | 2 ++ .../TemporalDensityCsvResultWriter.java | 33 +++++++++---------- .../r5/analyst/TemporalDensityResult.java | 9 ++++- .../r5/analyst/cluster/AnalysisWorker.java | 1 - 6 files changed, 37 insertions(+), 25 deletions(-) diff --git a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java index 1a62a21ff..b8859d9d5 100644 --- a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java +++ b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java @@ -164,11 +164,16 @@ public class AnalysisRequest { /** * Whether to include the number of opportunities reached during each minute of travel in results sent back * to the broker. Requires both an origin and destination pointset to be specified, and in the case of regional - * analyses the origins must be non-gridded. (Should be possible to make a grid as well.) + * analyses the origins must be non-gridded, and results will be collated to CSV. + * It should be possible to enable regional results for gridded origins as well. */ - public boolean opportunityTemporalDensity = false; + public boolean includeTemporalDensity = false; - public int dualAccessibilityOpportunityThreshold = 0; + /** + * If this is set to a value above zero, report the amount of time needed to reach the given number of + * opportunities from this origin (known technically as "dual accessibility"). + */ + public int dualAccessibilityThreshold = 0; /** @@ -276,8 +281,8 @@ public void populateTask (AnalysisWorkerTask task, UserPermissions userPermissio } } - task.includeTemporalDensity = opportunityTemporalDensity; - task.dualAccessibilityThreshold = dualAccessibilityOpportunityThreshold; + task.includeTemporalDensity = includeTemporalDensity; + task.dualAccessibilityThreshold = dualAccessibilityThreshold; } private EnumSet getEnumSetFromString (String s) { diff --git a/src/main/java/com/conveyal/analysis/results/CsvResultType.java b/src/main/java/com/conveyal/analysis/results/CsvResultType.java index 56612aafc..29446ba98 100644 --- a/src/main/java/com/conveyal/analysis/results/CsvResultType.java +++ b/src/main/java/com/conveyal/analysis/results/CsvResultType.java @@ -5,5 +5,5 @@ * do serve to enumerate the acceptable parameters coming over the HTTP API. */ public enum CsvResultType { - ACCESS, TIMES, PATHS, OPPORTUNITIES + ACCESS, TIMES, PATHS, TDENSITY } diff --git a/src/main/java/com/conveyal/analysis/results/MultiOriginAssembler.java b/src/main/java/com/conveyal/analysis/results/MultiOriginAssembler.java index d5c5094b2..638f83324 100644 --- a/src/main/java/com/conveyal/analysis/results/MultiOriginAssembler.java +++ b/src/main/java/com/conveyal/analysis/results/MultiOriginAssembler.java @@ -109,8 +109,10 @@ public MultiOriginAssembler (RegionalAnalysis regionalAnalysis, Job job, FileSto if (job.templateTask.recordAccessibility) { if (job.templateTask.originPointSet != null) { + // Freeform origins - create CSV regional analysis results resultWriters.add(new AccessCsvResultWriter(job.templateTask, fileStorage)); } else { + // Gridded origins - create gridded regional analysis results resultWriters.add(new MultiGridResultWriter(regionalAnalysis, job.templateTask, fileStorage)); } } diff --git a/src/main/java/com/conveyal/analysis/results/TemporalDensityCsvResultWriter.java b/src/main/java/com/conveyal/analysis/results/TemporalDensityCsvResultWriter.java index f9ed4472b..61279629c 100644 --- a/src/main/java/com/conveyal/analysis/results/TemporalDensityCsvResultWriter.java +++ b/src/main/java/com/conveyal/analysis/results/TemporalDensityCsvResultWriter.java @@ -24,7 +24,7 @@ public TemporalDensityCsvResultWriter(RegionalTask task, FileStorage fileStorage @Override public CsvResultType resultType () { - return CsvResultType.OPPORTUNITIES; + return CsvResultType.TDENSITY; } @Override @@ -59,33 +59,32 @@ protected void checkDimension (RegionalWorkResult workResult) { @Override public Iterable rowValues (RegionalWorkResult workResult) { - List row = new ArrayList<>(125); + List rows = new ArrayList<>(); String originId = task.originPointSet.getId(workResult.taskId); for (int d = 0; d < task.destinationPointSetKeys.length; d++) { - int[][] percentilesForDestPointset = workResult.accessibilityValues[d]; + double[][] percentilesForDestPointset = workResult.opportunitiesPerMinute[d]; for (int p = 0; p < task.percentiles.length; p++) { + List row = new ArrayList<>(125); row.add(originId); - row.add(task.destinationPointSets[d].name); + row.add(task.destinationPointSetKeys[d]); row.add(Integer.toString(p)); - int[] densitiesPerMinute = percentilesForDestPointset[p]; + // One density value for each of 120 minutes + double[] densitiesPerMinute = percentilesForDestPointset[p]; for (int m = 0; m < 120; m++) { row.add(Double.toString(densitiesPerMinute[m])); } - // Dual accessibility - { - int m = 0; - double sum = 0; - while (sum < dualThreshold) { - sum += densitiesPerMinute[m]; - m += 1; - } - row.add(Integer.toString(m >= 120 ? -1 : m)); + // One dual accessibility value + int m = 0; + double sum = 0; + while (sum < dualThreshold && m < 120) { + sum += densitiesPerMinute[m]; + m += 1; } + row.add(Integer.toString(m >= 120 ? -1 : m)); + rows.add(row.toArray(new String[row.size()])); } } - // List.of() or Arrays.asList() don't work without explicitly specifying the generic type because - // they interpret the String[] as varargs in the method signature. - return List.of(row.toArray(new String[0])); + return rows; } } diff --git a/src/main/java/com/conveyal/r5/analyst/TemporalDensityResult.java b/src/main/java/com/conveyal/r5/analyst/TemporalDensityResult.java index 5a4723204..1eda7445a 100644 --- a/src/main/java/com/conveyal/r5/analyst/TemporalDensityResult.java +++ b/src/main/java/com/conveyal/r5/analyst/TemporalDensityResult.java @@ -27,6 +27,7 @@ public class TemporalDensityResult { private final PointSet[] destinationPointSets; private final int nPercentiles; + private final int opportunityThreshold; // Externally visible fields for accumulating results @@ -44,6 +45,7 @@ public TemporalDensityResult(AnalysisWorkerTask task) { ); this.destinationPointSets = task.destinationPointSets; this.nPercentiles = task.percentiles.length; + this.opportunityThreshold = task.dualAccessibilityThreshold; this.opportunitiesPerMinute = new double[destinationPointSets.length][nPercentiles][120]; } @@ -66,7 +68,8 @@ public void recordOneTarget (int target, int[] travelTimePercentilesSeconds) { /** * Calculate "dual" accessibility from the accumulated temporal opportunity density array. * @param n the threshold quantity of opportunities - * @return the number of minutes it takes to reach n opportunities, for each destination set and percentile of travel time. + * @return the minimum whole number of minutes necessary to reach n opportunities, + * for each destination set and percentile of travel time. */ public int[][] minutesToReachOpportunities(int n) { int[][] result = new int[destinationPointSets.length][nPercentiles]; @@ -86,4 +89,8 @@ public int[][] minutesToReachOpportunities(int n) { return result; } + public int[][] minutesToReachOpportunities() { + return minutesToReachOpportunities(opportunityThreshold); + } + } diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java index 2104d16ee..2c35dac06 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java @@ -536,7 +536,6 @@ public static void addJsonToGrid ( } if (oneOriginResult.density != null) { jsonBlock.opportunitiesPerMinute = oneOriginResult.density.opportunitiesPerMinute; - LOG.info("Opportunities per minute: {}", oneOriginResult.density.opportunitiesPerMinute); } } LOG.debug("Travel time surface written, appending {}.", jsonBlock); From 75133370811f63875bf024d2d4be1a5353f521eb Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Thu, 17 Aug 2023 18:27:01 +0800 Subject: [PATCH 8/9] check for freeform origins before creating CSV --- .../conveyal/analysis/results/CsvResultWriter.java | 3 +++ .../analysis/results/MultiOriginAssembler.java | 12 +++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/analysis/results/CsvResultWriter.java b/src/main/java/com/conveyal/analysis/results/CsvResultWriter.java index e07abc2af..018931209 100644 --- a/src/main/java/com/conveyal/analysis/results/CsvResultWriter.java +++ b/src/main/java/com/conveyal/analysis/results/CsvResultWriter.java @@ -12,6 +12,8 @@ import java.io.FileWriter; import java.io.IOException; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; /** @@ -55,6 +57,7 @@ public abstract class CsvResultWriter extends BaseResultWriter implements Region */ CsvResultWriter (RegionalTask task, FileStorage fileStorage) throws IOException { super(fileStorage); + checkArgument(task.originPointSet != null, "CsvResultWriters require FreeFormPointSet origins."); super.prepare(task.jobId); this.fileName = task.jobId + "_" + resultType() +".csv"; BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(bufferFile)); diff --git a/src/main/java/com/conveyal/analysis/results/MultiOriginAssembler.java b/src/main/java/com/conveyal/analysis/results/MultiOriginAssembler.java index 638f83324..17fc4902a 100644 --- a/src/main/java/com/conveyal/analysis/results/MultiOriginAssembler.java +++ b/src/main/java/com/conveyal/analysis/results/MultiOriginAssembler.java @@ -126,7 +126,17 @@ public MultiOriginAssembler (RegionalAnalysis regionalAnalysis, Job job, FileSto } if (job.templateTask.includeTemporalDensity) { - resultWriters.add(new TemporalDensityCsvResultWriter(job.templateTask, fileStorage)); + if (job.templateTask.originPointSet == null) { + // Gridded origins. The full temporal density information is probably too voluminous to be useful. + // We might want to record a grid of dual accessibility values, but this will require some serious + // refactoring of the GridResultWriter. + // if (job.templateTask.dualAccessibilityThreshold > 0) { ... } + throw new IllegalArgumentException("Temporal density of opportunities cannot be recorded for gridded origin points."); + } else { + // Freeform origins. + // Output includes temporal density of opportunities and optionally dual accessibility. + resultWriters.add(new TemporalDensityCsvResultWriter(job.templateTask, fileStorage)); + } } checkArgument(job.templateTask.makeTauiSite || notNullOrEmpty(resultWriters), From 98b741a2e2d101cac55d52a06ed88ccd021bbe52 Mon Sep 17 00:00:00 2001 From: Andrew Byrd Date: Tue, 26 Sep 2023 20:41:07 +0200 Subject: [PATCH 9/9] update javadoc --- .../conveyal/r5/analyst/TemporalDensityResult.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/conveyal/r5/analyst/TemporalDensityResult.java b/src/main/java/com/conveyal/r5/analyst/TemporalDensityResult.java index 1eda7445a..60956048c 100644 --- a/src/main/java/com/conveyal/r5/analyst/TemporalDensityResult.java +++ b/src/main/java/com/conveyal/r5/analyst/TemporalDensityResult.java @@ -7,19 +7,19 @@ import static com.conveyal.r5.profile.FastRaptorWorker.UNREACHED; /** - * An instance of this is included in a OneOriginResult for reporting the nearest N destinations. If we use more than - * one destination point set they are already constrained to all be aligned with the same number of destinations. + * An instance of this is included in a OneOriginResult for reporting how many opportunities are encountered during each + * minute of travel. If we use more than one destination point set they are already constrained to all be aligned with + * the same number of destinations. * * The data retained here feed into three different kinds of results: "Dual" accessibility (the number of opportunities - * reached in a given number of minutes of travel time); temporal opportunity density (akin to a probability density + * reached in a given number of minutes of travel time); temporal opportunity density (analogous to a probability density * function, how many opportunities are encountered during each minute of travel, whose integral is the cumulative - * accessibility curve); and the nearest one or more opportunities to a given origin. - * (expand from comments on https://github.com/conveyal/r5/pull/884) + * accessibility curve). * * Originally this class was tracking the identity of the N nearest points rather than just binning them by travel time. * This is more efficient in cases where N is small, and allows retaining the one-second resolution. However currently * there does not seem to be much demand among users for this level of detail, so it has been removed in the interest - * of simplicity and maintainability. + * of simplicity and maintainability. See issue 884 for more comments on implementation trade-offs. */ public class TemporalDensityResult { @@ -34,7 +34,6 @@ public class TemporalDensityResult { /** * The temporal density of opportunities. For each destination set, for each percentile, for each minute of * travel from 0 to 120, the number of opportunities reached in travel times from i (inclusive) to i+1 (exclusive). - * */ public final double[][][] opportunitiesPerMinute;