diff --git a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java index 10d326656..b8859d9d5 100644 --- a/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java +++ b/src/main/java/com/conveyal/analysis/models/AnalysisRequest.java @@ -161,6 +161,21 @@ public class AnalysisRequest { */ public ChaosParameters injectFault; + /** + * Whether to include the number of opportunities reached during each minute of travel in results sent back + * to the broker. Requires both an origin and destination pointset to be specified, and in the case of regional + * analyses the origins must be non-gridded, and results will be collated to CSV. + * It should be possible to enable regional results for gridded origins as well. + */ + public boolean includeTemporalDensity = false; + + /** + * If this is set to a value above zero, report the amount of time needed to reach the given number of + * opportunities from this origin (known technically as "dual accessibility"). + */ + public int dualAccessibilityThreshold = 0; + + /** * Create the R5 `Scenario` from this request. */ @@ -265,6 +280,9 @@ public void populateTask (AnalysisWorkerTask task, UserPermissions userPermissio throw new IllegalArgumentException("Must be admin user to inject faults."); } } + + task.includeTemporalDensity = includeTemporalDensity; + task.dualAccessibilityThreshold = dualAccessibilityThreshold; } private EnumSet getEnumSetFromString (String s) { diff --git a/src/main/java/com/conveyal/analysis/results/CsvResultType.java b/src/main/java/com/conveyal/analysis/results/CsvResultType.java index cdf60e410..29446ba98 100644 --- a/src/main/java/com/conveyal/analysis/results/CsvResultType.java +++ b/src/main/java/com/conveyal/analysis/results/CsvResultType.java @@ -5,5 +5,5 @@ * do serve to enumerate the acceptable parameters coming over the HTTP API. */ public enum CsvResultType { - ACCESS, TIMES, PATHS + ACCESS, TIMES, PATHS, TDENSITY } diff --git a/src/main/java/com/conveyal/analysis/results/CsvResultWriter.java b/src/main/java/com/conveyal/analysis/results/CsvResultWriter.java index e07abc2af..018931209 100644 --- a/src/main/java/com/conveyal/analysis/results/CsvResultWriter.java +++ b/src/main/java/com/conveyal/analysis/results/CsvResultWriter.java @@ -12,6 +12,8 @@ import java.io.FileWriter; import java.io.IOException; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; /** @@ -55,6 +57,7 @@ public abstract class CsvResultWriter extends BaseResultWriter implements Region */ CsvResultWriter (RegionalTask task, FileStorage fileStorage) throws IOException { super(fileStorage); + checkArgument(task.originPointSet != null, "CsvResultWriters require FreeFormPointSet origins."); super.prepare(task.jobId); this.fileName = task.jobId + "_" + resultType() +".csv"; BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(bufferFile)); diff --git a/src/main/java/com/conveyal/analysis/results/MultiOriginAssembler.java b/src/main/java/com/conveyal/analysis/results/MultiOriginAssembler.java index 3bbd5915f..17fc4902a 100644 --- a/src/main/java/com/conveyal/analysis/results/MultiOriginAssembler.java +++ b/src/main/java/com/conveyal/analysis/results/MultiOriginAssembler.java @@ -109,8 +109,10 @@ public MultiOriginAssembler (RegionalAnalysis regionalAnalysis, Job job, FileSto if (job.templateTask.recordAccessibility) { if (job.templateTask.originPointSet != null) { + // Freeform origins - create CSV regional analysis results resultWriters.add(new AccessCsvResultWriter(job.templateTask, fileStorage)); } else { + // Gridded origins - create gridded regional analysis results resultWriters.add(new MultiGridResultWriter(regionalAnalysis, job.templateTask, fileStorage)); } } @@ -123,6 +125,20 @@ public MultiOriginAssembler (RegionalAnalysis regionalAnalysis, Job job, FileSto resultWriters.add(new PathCsvResultWriter(job.templateTask, fileStorage)); } + if (job.templateTask.includeTemporalDensity) { + if (job.templateTask.originPointSet == null) { + // Gridded origins. The full temporal density information is probably too voluminous to be useful. + // We might want to record a grid of dual accessibility values, but this will require some serious + // refactoring of the GridResultWriter. + // if (job.templateTask.dualAccessibilityThreshold > 0) { ... } + throw new IllegalArgumentException("Temporal density of opportunities cannot be recorded for gridded origin points."); + } else { + // Freeform origins. + // Output includes temporal density of opportunities and optionally dual accessibility. + resultWriters.add(new TemporalDensityCsvResultWriter(job.templateTask, fileStorage)); + } + } + checkArgument(job.templateTask.makeTauiSite || notNullOrEmpty(resultWriters), "A non-Taui regional analysis should always create at least one grid or CSV file."); diff --git a/src/main/java/com/conveyal/analysis/results/TemporalDensityCsvResultWriter.java b/src/main/java/com/conveyal/analysis/results/TemporalDensityCsvResultWriter.java new file mode 100644 index 000000000..61279629c --- /dev/null +++ b/src/main/java/com/conveyal/analysis/results/TemporalDensityCsvResultWriter.java @@ -0,0 +1,90 @@ +package com.conveyal.analysis.results; + +import com.conveyal.file.FileStorage; +import com.conveyal.r5.analyst.cluster.RegionalTask; +import com.conveyal.r5.analyst.cluster.RegionalWorkResult; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * This handles collating regional results into CSV files containing temporal opportunity density + * (number of opportunities reached in each one-minute interval, the derivative of step-function accessibility) + * as well as "dual" accessibility (the amount of time needed to reach n opportunities). + */ +public class TemporalDensityCsvResultWriter extends CsvResultWriter { + + private final int dualThreshold; + + public TemporalDensityCsvResultWriter(RegionalTask task, FileStorage fileStorage) throws IOException { + super(task, fileStorage); + dualThreshold = task.dualAccessibilityThreshold; + } + + @Override + public CsvResultType resultType () { + return CsvResultType.TDENSITY; + } + + @Override + public String[] columnHeaders () { + List headers = new ArrayList<>(); + // The ids of the freeform origin point and destination set + headers.add("originId"); + headers.add("destId"); + headers.add("percentile"); + for (int m = 0; m < 120; m += 1) { + // The opportunity density over travel minute m + headers.add(Integer.toString(m)); + } + // The number of minutes needed to reach d destination opportunities + headers.add("D" + dualThreshold); + return headers.toArray(new String[0]); + } + + @Override + protected void checkDimension (RegionalWorkResult workResult) { + checkDimension( + workResult, "destination pointsets", + workResult.opportunitiesPerMinute.length, task.destinationPointSetKeys.length + ); + for (double[][] percentilesForPointset : workResult.opportunitiesPerMinute) { + checkDimension(workResult, "percentiles", percentilesForPointset.length, task.percentiles.length); + for (double[] minutesForPercentile : percentilesForPointset) { + checkDimension(workResult, "minutes", minutesForPercentile.length, 120); + } + } + } + + @Override + public Iterable rowValues (RegionalWorkResult workResult) { + List rows = new ArrayList<>(); + String originId = task.originPointSet.getId(workResult.taskId); + for (int d = 0; d < task.destinationPointSetKeys.length; d++) { + double[][] percentilesForDestPointset = workResult.opportunitiesPerMinute[d]; + for (int p = 0; p < task.percentiles.length; p++) { + List row = new ArrayList<>(125); + row.add(originId); + row.add(task.destinationPointSetKeys[d]); + row.add(Integer.toString(p)); + // One density value for each of 120 minutes + double[] densitiesPerMinute = percentilesForDestPointset[p]; + for (int m = 0; m < 120; m++) { + row.add(Double.toString(densitiesPerMinute[m])); + } + // One dual accessibility value + int m = 0; + double sum = 0; + while (sum < dualThreshold && m < 120) { + sum += densitiesPerMinute[m]; + m += 1; + } + row.add(Integer.toString(m >= 120 ? -1 : m)); + rows.add(row.toArray(new String[row.size()])); + } + } + return rows; + } + +} diff --git a/src/main/java/com/conveyal/r5/OneOriginResult.java b/src/main/java/com/conveyal/r5/OneOriginResult.java index 6c721d41e..090cdd9a0 100644 --- a/src/main/java/com/conveyal/r5/OneOriginResult.java +++ b/src/main/java/com/conveyal/r5/OneOriginResult.java @@ -1,6 +1,7 @@ package com.conveyal.r5; import com.conveyal.r5.analyst.AccessibilityResult; +import com.conveyal.r5.analyst.TemporalDensityResult; import com.conveyal.r5.analyst.cluster.PathResult; import com.conveyal.r5.analyst.cluster.TravelTimeResult; @@ -20,10 +21,18 @@ public class OneOriginResult { public final PathResult paths; - public OneOriginResult(TravelTimeResult travelTimes, AccessibilityResult accessibility, PathResult paths) { + public final TemporalDensityResult density; + + public OneOriginResult( + TravelTimeResult travelTimes, + AccessibilityResult accessibility, + PathResult paths, + TemporalDensityResult density + ) { this.travelTimes = travelTimes; this.accessibility = accessibility; this.paths = paths; + this.density = density; } } diff --git a/src/main/java/com/conveyal/r5/analyst/TemporalDensityResult.java b/src/main/java/com/conveyal/r5/analyst/TemporalDensityResult.java new file mode 100644 index 000000000..60956048c --- /dev/null +++ b/src/main/java/com/conveyal/r5/analyst/TemporalDensityResult.java @@ -0,0 +1,95 @@ +package com.conveyal.r5.analyst; + +import com.conveyal.r5.analyst.cluster.AnalysisWorkerTask; +import com.google.common.base.Preconditions; + +import static com.conveyal.r5.common.Util.notNullOrEmpty; +import static com.conveyal.r5.profile.FastRaptorWorker.UNREACHED; + +/** + * An instance of this is included in a OneOriginResult for reporting how many opportunities are encountered during each + * minute of travel. If we use more than one destination point set they are already constrained to all be aligned with + * the same number of destinations. + * + * The data retained here feed into three different kinds of results: "Dual" accessibility (the number of opportunities + * reached in a given number of minutes of travel time); temporal opportunity density (analogous to a probability density + * function, how many opportunities are encountered during each minute of travel, whose integral is the cumulative + * accessibility curve). + * + * Originally this class was tracking the identity of the N nearest points rather than just binning them by travel time. + * This is more efficient in cases where N is small, and allows retaining the one-second resolution. However currently + * there does not seem to be much demand among users for this level of detail, so it has been removed in the interest + * of simplicity and maintainability. See issue 884 for more comments on implementation trade-offs. + */ +public class TemporalDensityResult { + + // Internal state fields + + private final PointSet[] destinationPointSets; + private final int nPercentiles; + private final int opportunityThreshold; + + // Externally visible fields for accumulating results + + /** + * The temporal density of opportunities. For each destination set, for each percentile, for each minute of + * travel from 0 to 120, the number of opportunities reached in travel times from i (inclusive) to i+1 (exclusive). + */ + public final double[][][] opportunitiesPerMinute; + + public TemporalDensityResult(AnalysisWorkerTask task) { + Preconditions.checkArgument( + notNullOrEmpty(task.destinationPointSets), + "Dual accessibility requires at least one destination pointset." + ); + this.destinationPointSets = task.destinationPointSets; + this.nPercentiles = task.percentiles.length; + this.opportunityThreshold = task.dualAccessibilityThreshold; + this.opportunitiesPerMinute = new double[destinationPointSets.length][nPercentiles][120]; + } + + public void recordOneTarget (int target, int[] travelTimePercentilesSeconds) { + // Increment histogram bin for the number of minutes of travel by the number of opportunities at the target. + for (int d = 0; d < destinationPointSets.length; d++) { + PointSet dps = destinationPointSets[d]; + for (int p = 0; p < nPercentiles; p++) { + if (travelTimePercentilesSeconds[p] == UNREACHED) { + break; // If any percentile is unreached, all higher ones are also unreached. + } + int m = travelTimePercentilesSeconds[p] / 60; + if (m <= 120) { + opportunitiesPerMinute[d][p][m] += dps.getOpportunityCount(target); + } + } + } + } + + /** + * Calculate "dual" accessibility from the accumulated temporal opportunity density array. + * @param n the threshold quantity of opportunities + * @return the minimum whole number of minutes necessary to reach n opportunities, + * for each destination set and percentile of travel time. + */ + public int[][] minutesToReachOpportunities(int n) { + int[][] result = new int[destinationPointSets.length][nPercentiles]; + for (int d = 0; d < destinationPointSets.length; d++) { + for (int p = 0; p < nPercentiles; p++) { + result[d][p] = -1; + double count = 0; + for (int m = 0; m < 120; m++) { + count += opportunitiesPerMinute[d][p][m]; + if (count >= n) { + result[d][p] = m + 1; + break; + } + } + } + } + return result; + } + + public int[][] minutesToReachOpportunities() { + return minutesToReachOpportunities(opportunityThreshold); + } + +} diff --git a/src/main/java/com/conveyal/r5/analyst/TravelTimeReducer.java b/src/main/java/com/conveyal/r5/analyst/TravelTimeReducer.java index 4e1ca30bc..15ce02012 100644 --- a/src/main/java/com/conveyal/r5/analyst/TravelTimeReducer.java +++ b/src/main/java/com/conveyal/r5/analyst/TravelTimeReducer.java @@ -46,6 +46,9 @@ public class TravelTimeReducer { /** Retains the paths to one or all destinations, for recording in CSV or reporting in the UI. */ private PathResult pathResult = null; + /** Represents how many destinations are reached in each minute of travel from this origin. */ + private TemporalDensityResult temporalDensityResult = null; + /** If we are calculating accessibility, the PointSets containing opportunities. */ private PointSet[] destinationPointSets; @@ -121,6 +124,8 @@ public TravelTimeReducer (AnalysisWorkerTask task, TransportNetwork network) { this.destinationPointSets = task.destinationPointSets; if (task instanceof TravelTimeSurfaceTask) { calculateTravelTimes = true; + // In single-point analyses, destination pointsets may be missing if the user has not selected one in the + // UI, or if the user has selected the step decay function instead of one of the other decay functions. calculateAccessibility = notNullOrEmpty(task.destinationPointSets); } else { // Maybe we should define recordAccessibility and recordTimes on the common superclass AnalysisWorkerTask. @@ -140,6 +145,9 @@ public TravelTimeReducer (AnalysisWorkerTask task, TransportNetwork network) { if (task.includePathResults) { pathResult = new PathResult(task, network.transitLayer); } + if (task.includeTemporalDensity) { + temporalDensityResult = new TemporalDensityResult(task); + } // Validate and copy the travel time cutoffs, converting them to seconds to avoid repeated multiplication // in tight loops. Also find the points where the decay function reaches zero for these cutoffs. @@ -271,6 +279,9 @@ private void recordTravelTimePercentilesForTarget (int target, int[] travelTimeP } } } + if (temporalDensityResult != null) { + temporalDensityResult.recordOneTarget(target, travelTimePercentilesSeconds); + } } /** @@ -329,7 +340,7 @@ private int convertToMinutes (int timeSeconds) { * origin point is not connected to the street network. */ public OneOriginResult finish () { - return new OneOriginResult(travelTimeResult, accessibilityResult, pathResult); + return new OneOriginResult(travelTimeResult, accessibilityResult, pathResult, temporalDensityResult); } /** diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java index aad7da99d..2c35dac06 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorker.java @@ -13,6 +13,7 @@ import com.conveyal.r5.analyst.TravelTimeComputer; import com.conveyal.r5.analyst.error.TaskError; import com.conveyal.r5.common.JsonUtilities; +import com.conveyal.r5.transit.TransitLayer; import com.conveyal.r5.transit.TransportNetwork; import com.conveyal.r5.transit.TransportNetworkCache; import com.conveyal.r5.transitive.TransitiveNetwork; @@ -344,10 +345,10 @@ private byte[] singlePointResultToBinary ( timeGridWriter.writeToDataOutput(new LittleEndianDataOutputStream(byteArrayOutputStream)); addJsonToGrid( byteArrayOutputStream, - oneOriginResult.accessibility, + oneOriginResult, transportNetwork.scenarioApplicationWarnings, transportNetwork.scenarioApplicationInfo, - oneOriginResult.paths != null ? new PathResultSummary(oneOriginResult.paths, transportNetwork.transitLayer) : null + transportNetwork.transitLayer ); } // Single-point tasks don't have a job ID. For now, we'll categorize them by scenario ID. @@ -457,7 +458,7 @@ protected void handleOneRegionalTask (RegionalTask task) throws Throwable { // progress. This avoids crashing the backend by sending back massive 2 million element travel times // that have already been written to S3, and throwing exceptions on old backends that can't deal with // null AccessibilityResults. - oneOriginResult = new OneOriginResult(null, new AccessibilityResult(task), null); + oneOriginResult = new OneOriginResult(null, new AccessibilityResult(task), null, null); } // Accumulate accessibility results, which will be returned to the backend in batches. @@ -489,6 +490,8 @@ public static class GridJsonBlock { public PathResultSummary pathSummaries; + public double[][][] opportunitiesPerMinute; + @Override public String toString () { return String.format( @@ -508,24 +511,33 @@ public String toString () { * and no serious errors, we use a success error code. * TODO distinguish between warnings and errors - we already distinguish between info and warnings. * This could be turned into a GridJsonBlock constructor, with the JSON writing code in an instance method. + * Note that this is very similar to the RegionalWorkResult constructor, and some duplication of logic could be + * achieved by somehow merging the two. */ public static void addJsonToGrid ( OutputStream outputStream, - AccessibilityResult accessibilityResult, + OneOriginResult oneOriginResult, List scenarioApplicationWarnings, List scenarioApplicationInfo, - PathResultSummary pathResult + TransitLayer transitLayer // Only used if oneOriginResult contains paths, can be null otherwise ) throws IOException { var jsonBlock = new GridJsonBlock(); jsonBlock.scenarioApplicationInfo = scenarioApplicationInfo; jsonBlock.scenarioApplicationWarnings = scenarioApplicationWarnings; - if (accessibilityResult != null) { - // Due to the application of distance decay functions, we may want to make the shift to non-integer - // accessibility values (especially for cases where there are relatively few opportunities across the whole - // study area). But we'd need to control the number of decimal places serialized into the JSON. - jsonBlock.accessibility = accessibilityResult.getIntValues(); + if (oneOriginResult != null) { + if (oneOriginResult.accessibility != null) { + // Due to the application of distance decay functions, we may want to make the shift to non-integer + // accessibility values (especially for cases where there are relatively few opportunities across the whole + // study area). But we'd need to control the number of decimal places serialized into the JSON. + jsonBlock.accessibility = oneOriginResult.accessibility.getIntValues(); + } + if (oneOriginResult.paths != null) { + jsonBlock.pathSummaries = new PathResultSummary(oneOriginResult.paths, transitLayer); + } + if (oneOriginResult.density != null) { + jsonBlock.opportunitiesPerMinute = oneOriginResult.density.opportunitiesPerMinute; + } } - jsonBlock.pathSummaries = pathResult; LOG.debug("Travel time surface written, appending {}.", jsonBlock); // We could do this when setting up the Spark handler, supplying writeValue as the response transformer // But then you also have to handle the case where you are returning raw bytes. diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorkerTask.java b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorkerTask.java index d1e88f78c..f3305e280 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorkerTask.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/AnalysisWorkerTask.java @@ -99,6 +99,20 @@ public abstract class AnalysisWorkerTask extends ProfileRequest { */ public boolean includePathResults = false; + /** + * Whether to include the number of opportunities reached during each minute of travel in results sent back + * to the broker. Requires both an origin and destination pointset to be specified, and in the case of regional + * analyses the origins must be non-gridded, and results will be collated to CSV. + * It should be possible to enable regional results for gridded origins as well. + */ + public boolean includeTemporalDensity = false; + + /** + * If this is set to a value above zero, report the amount of time needed to reach the given number of + * opportunities from this origin (known technically as "dual accessibility"). + */ + public int dualAccessibilityThreshold = 0; + /** Whether to build a histogram of travel times to each destination, generally used in testing and debugging. */ public boolean recordTravelTimeHistograms = false; diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java b/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java index 39a650d3e..d15531c03 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/RegionalWorkResult.java @@ -37,6 +37,12 @@ public class RegionalWorkResult { */ public int[][][] accessibilityValues; + /** + * The temporal density of opportunities - how many are reached during each minute of travel. + * Quantities of opportunities for each [destinationGrid, percentile, minute]. + */ + public double[][][] opportunitiesPerMinute; + /** * If this field is non-null, the worker is reporting an error that compromises the quality of the result at this * origin point, and potentially for then entire regional analysis. Put into a Set on backend since all workers @@ -59,6 +65,7 @@ public RegionalWorkResult(OneOriginResult result, RegionalTask task) { this.travelTimeValues = result.travelTimes == null ? null : result.travelTimes.values; this.accessibilityValues = result.accessibility == null ? null : result.accessibility.getIntValues(); this.pathResult = result.paths == null ? null : result.paths.summarizeIterations(PathResult.Stat.MINIMUM); + this.opportunitiesPerMinute = result.density == null ? null : result.density.opportunitiesPerMinute; // TODO checkTravelTimeInvariants, checkAccessibilityInvariants to verify that values are monotonically increasing } diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/TravelTimeResult.java b/src/main/java/com/conveyal/r5/analyst/cluster/TravelTimeResult.java index c79f23ab8..c851afcac 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/TravelTimeResult.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/TravelTimeResult.java @@ -37,6 +37,9 @@ public class TravelTimeResult { * This is optional, generally used only for debugging and testing, so may be null if no histograms are recorded. * Since this is used for debugging and testing it might even be interesting to do it at one-second resolution or * just store every travel time, but binning makes it easy to compare with probability distributions. + * Note that this is one histogram _per target_ showing on how many iterations each travel time is the fastest, + * _not_ one histogram per origin/percentile showing how many destinations are reached at each travel time. The + * latter is essentially the discrete derivative of step-function accessibility and is tracked elsewhere (TemporalDensityResult). */ int[][] histograms; diff --git a/src/main/java/com/conveyal/r5/common/Util.java b/src/main/java/com/conveyal/r5/common/Util.java index ddfce26d5..c79e1c168 100644 --- a/src/main/java/com/conveyal/r5/common/Util.java +++ b/src/main/java/com/conveyal/r5/common/Util.java @@ -3,6 +3,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Map; +import java.util.function.Supplier; public abstract class Util { @@ -66,4 +67,19 @@ public static int[] newIntArray (int length, int defaultValue) { return array; } + /** + * Convenience method to create a 2D array and fill it immediately with new instances of a class. + * The supplier can be a method reference to a constructor like ToBeInstantiated::new, and the returned + * array will be of that type. + */ + public static T[][] newObjectArray (int d1, int d2, Supplier supplier) { + T[][] result = (T[][]) new Object[d1][d2]; + for (int x = 0; x < d1; x++) { + for (int y = 0; y < d2; y++) { + result[x][y] = supplier.get(); + } + } + return result; + } + }