Merge pull request #862 from conveyal/prebuild-non-walk

Pre-build linkages and egress tables for additional street modes via TransportNetworkConfig

src/main/java/com/conveyal/r5/analyst/cluster/TransportNetworkConfig.java

@@ -4,11 +4,13 @@
 import com.conveyal.r5.analyst.scenario.Modification;
 import com.conveyal.r5.analyst.scenario.RasterCost;
 import com.conveyal.r5.analyst.scenario.ShapefileLts;
+import com.conveyal.r5.profile.StreetMode;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 
 import java.util.List;
+import java.util.Set;
 
 /**
  * All inputs and options that describe how to build a particular transport network (except the serialization version).
@@ -39,4 +41,17 @@ public class TransportNetworkConfig {
     /** A list of _R5_ modifications to apply during network build. May be null. */
     public List<Modification> modifications;
 
+    /**
+     * Additional modes other than walk for which to pre-build large data structures (grid linkage and cost tables).
+     * When building a network, by default we build distance tables from transit stops to street vertices, to which we
+     * connect a grid covering the entire street network at the default zoom level. By default we do this only for the
+     * walk mode. Pre-building and serializing equivalent data structures for other modes allows workers to start up
+     * much faster in regional analyses. The work need only be done once when the first single-point worker to builds
+     * the network. Otherwise, hundreds of workers will each have to build these tables every time they start up.
+     * Some scenarios, such as those that affect the street layer, may still be slower to apply for modes listed here
+     * because some intermediate data (stop-to-vertex tables) are only retained for the walk mode. If this proves to be
+     * a problem it is a candidate for future optimization.
+     */
+    public Set<StreetMode> buildGridsForModes;
+
 }

src/main/java/com/conveyal/r5/streets/EgressCostTable.java

@@ -209,7 +209,7 @@ public EgressCostTable (LinkedPointSet linkedPointSet, ProgressListener progress
             rebuildZone = linkedPointSet.streetLayer.scenarioEdgesBoundingGeometry(linkingDistanceLimitMeters);
         }
 
-        LOG.info("Creating EgressCostTables from each transit stop to PointSet points.");
+        LOG.info("Creating EgressCostTables from each transit stop to PointSet points for mode {}.", streetMode);
         if (rebuildZone != null) {
             LOG.info("Selectively computing tables for only those stops that might be affected by the scenario.");
         }
@@ -232,9 +232,9 @@ public EgressCostTable (LinkedPointSet linkedPointSet, ProgressListener progress
         progressListener.beginTask(taskDescription, nStops);
 
         final LambdaCounter computeCounter = new LambdaCounter(LOG, nStops, computeLogFrequency,
-                "Computed new stop -> point tables for {} of {} transit stops.");
+                String.format("Computed new stop-to-point tables from {} of {} transit stops for mode %s.", streetMode));
         final LambdaCounter copyCounter = new LambdaCounter(LOG, nStops, copyLogFrequency,
-                "Copied unchanged stop -> point tables for {} of {} transit stops.");
+                String.format("Copied unchanged stop-to-point tables from {} of {} transit stops for mode %s.", streetMode));
         // Create a distance table from each transit stop to the points in this PointSet in parallel.
         // Each table is a flattened 2D array. Two values for each point reachable from this stop: (pointIndex, cost)
         // When applying a scenario, keep the existing distance table for those stops that could not be affected.
@@ -262,16 +262,16 @@ public EgressCostTable (LinkedPointSet linkedPointSet, ProgressListener progress
             GeometryUtils.expandEnvelopeFixed(envelopeAroundStop, linkingDistanceLimitMeters);
 
             if (streetMode == StreetMode.WALK) {
-                // Walking distances from stops to street vertices are saved in the TransitLayer.
-                // Get the pre-computed walking distance table from the stop to the street vertices,
-                // then extend that table out from the street vertices to the points in this PointSet.
-                // TODO reuse the code that computes the walk tables at TransitLayer.buildOneDistanceTable() rather than
-                //      duplicating it below for other modes.
+                // Distances from stops to street vertices are saved in the TransitLayer, but only for the walk mode.
+                // Get the pre-computed walking distance table from the stop to the street vertices, then extend that
+                // table out from the street vertices to the points in this PointSet. It may be possible to reuse the
+                // code that pre-computes walk tables at TransitLayer.buildOneDistanceTable() rather than duplicating
+                // it below for other (non-walk) modes.
                 TIntIntMap distanceTableToVertices = transitLayer.stopToVertexDistanceTables.get(stopIndex);
                 return distanceTableToVertices == null ? null :
                         linkedPointSet.extendDistanceTableToPoints(distanceTableToVertices, envelopeAroundStop);
             } else {
-
+                // For non-walk modes perform a search from each stop, as stop-to-vertex tables are not precomputed.
                 Geometry egressArea = null;
 
                 // If a pickup delay modification is present for this street mode, egressStopDelaysSeconds is
@@ -301,14 +301,14 @@ public EgressCostTable (LinkedPointSet linkedPointSet, ProgressListener progress
                     LOG.warn("Stop unlinked, cannot build distance table: {}", stopIndex);
                     return null;
                 }
-                // TODO setting the origin point of the router to the stop vertex does not work.
-                // This is probably because link edges do not allow car traversal. We could traverse them.
-                // As a stopgap we perform car linking at the geographic coordinate of the stop.
+                // Setting the origin point of the router to the stop vertex (as follows) does not work.
                 // sr.setOrigin(vertexId);
+                // This is probably because link edges do not allow car traversal. We could traverse them.
+                // As a workaround we perform car linking at the geographic coordinate of the stop.
                 VertexStore.Vertex vertex = linkedPointSet.streetLayer.vertexStore.getCursor(vertexId);
                 sr.setOrigin(vertex.getLat(), vertex.getLon());
 
-                // WALK is handled above, this block is exhaustively handling all other modes.
+                // WALK is handled in the if clause above, this else block is exhaustively handling all other modes.
                 if (streetMode == StreetMode.BICYCLE) {
                     sr.distanceLimitMeters = linkingDistanceLimitMeters;
                 } else if (streetMode == StreetMode.CAR) {

src/main/java/com/conveyal/r5/streets/LinkedPointSet.java

@@ -135,7 +135,7 @@ public class LinkedPointSet implements Serializable {
      *                    the same pointSet and streetMode as the preceding arguments.
      */
     public LinkedPointSet (PointSet pointSet, StreetLayer streetLayer, StreetMode streetMode, LinkedPointSet baseLinkage) {
-        LOG.info("Linking pointset to street network...");
+        LOG.info("Linking pointset to street network for mode {}...", streetMode);
         this.pointSet = pointSet;
         this.streetLayer = streetLayer;
         this.streetMode = streetMode;
@@ -301,7 +301,7 @@ public synchronized EgressCostTable getEgressCostTable () {
      */
     private void linkPointsToStreets (boolean all) {
         LambdaCounter linkCounter = new LambdaCounter(LOG, pointSet.featureCount(), 10000,
-                "Linked {} of {} PointSet points to streets.");
+                String.format("Linked {} of {} PointSet points to streets for mode %s.", streetMode));
 
         // Construct a geometry around any edges added by the scenario, or null if there are no added edges.
         // As it is derived from edge geometries this is a fixed-point geometry and must be intersected with the same.

src/main/java/com/conveyal/r5/transit/TransitLayer.java

@@ -535,16 +535,18 @@ public void rebuildTransientIndexes () {
     }
 
     /**
-     * Run a distance-constrained street search from every transit stop in the graph.
+     * Run a distance-constrained street search from every transit stop in the graph using the walk mode.
      * Store the distance to every reachable street vertex for each of these origin stops.
      * If a scenario has been applied, we need to build tables for any newly created stops and any stops within
      * transfer distance or access/egress distance of those new stops. In that case a rebuildZone geometry should be
      * supplied. If rebuildZone is null, a complete rebuild of all tables will occur for all stops.
+     * Note, this rebuilds for the WALK MODE ONLY. The network only has a field for retaining walk distance tables.
+     * This is a candidate for optimization if car or bicycle scenarios are slow to apply.
      * @param rebuildZone the zone within which to rebuild tables in FIXED-POINT DEGREES, or null to build all tables.
      */
     public void buildDistanceTables(Geometry rebuildZone) {
 
-        LOG.info("Finding distances from transit stops to street vertices.");
+        LOG.info("Pre-computing distances from transit stops to street vertices (WALK mode only).");
         if (rebuildZone != null) {
             LOG.info("Selectively finding distances for only those stops potentially affected by scenario application.");
         }

src/main/java/com/conveyal/r5/transit/TransportNetwork.java

@@ -25,6 +25,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.stream.Stream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
@@ -263,14 +264,14 @@ public static InputFileType forFile(File file) {
     }
 
     /**
-     * For Analysis purposes, build an efficient implicit grid PointSet for this TransportNetwork. Then, for any modes
-     * supplied, we also build a linkage that is held permanently in the GridPointSet. This method is called when a
-     * network is first built.
-     * The resulting grid PointSet will cover the entire street network layer of this TransportNetwork, which should
-     * include every point we can route from or to. Any other destination grid (for the same mode, walking) can be made
-     * as a subset of this one since it includes every potentially accessible point.
+     * Build a grid PointSet covering the entire street network layer of this TransportNetwork, which should include
+     * every point we can route from or to. Then for all requested modes build a linkage that is held in the
+     * GridPointSet. This method is called when a network is first built so these linkages are serialized with it.
+     * Any other destination grid (at least for the same modes) can be made as a subset of this one since it includes
+     * every potentially accessible point. Destination grids for other modes will be made on demand, which is a slow
+     * operation that can occupy hundreds of workers for long periods of time when a regional analysis starts up.
      */
-    public void rebuildLinkedGridPointSet(StreetMode... modes) {
+    public void rebuildLinkedGridPointSet(Iterable<StreetMode> modes) {
         if (fullExtentGridPointSet != null) {
             throw new RuntimeException("Linked grid pointset was built more than once.");
         }
@@ -280,6 +281,10 @@ public void rebuildLinkedGridPointSet(StreetMode... modes) {
         }
     }
 
+    public void rebuildLinkedGridPointSet(StreetMode... modes) {
+        rebuildLinkedGridPointSet(Set.of(modes));
+    }
+
     //TODO: add transit stops to envelope
     public Envelope getEnvelope() {
         return streetLayer.getEnvelope();

src/main/java/com/conveyal/r5/transit/TransportNetworkCache.java

@@ -19,6 +19,7 @@
 import com.conveyal.r5.streets.StreetLayer;
 import com.github.benmanes.caffeine.cache.Caffeine;
 import com.github.benmanes.caffeine.cache.LoadingCache;
+import com.google.common.collect.Sets;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -158,28 +159,52 @@ private static FileStorageKey getR5NetworkFileStorageKey (String networkId) {
         return new FileStorageKey(BUNDLES, getR5NetworkFilename(networkId));
     }
 
+    /** @return the network configuration (AKA manifest) for the given network ID, or null if no config file exists. */
+    private TransportNetworkConfig loadNetworkConfig (String networkId) {
+        FileStorageKey configFileKey = new FileStorageKey(BUNDLES, getNetworkConfigFilename(networkId));
+        if (!fileStorage.exists(configFileKey)) {
+            return null;
+        }
+        File configFile = fileStorage.getFile(configFileKey);
+        try {
+            // Use lenient mapper to mimic behavior in objectFromRequestBody.
+            return JsonUtilities.lenientObjectMapper.readValue(configFile, TransportNetworkConfig.class);
+        } catch (IOException e) {
+            throw new RuntimeException("Error reading TransportNetworkConfig. Does it contain new unrecognized fields?", e);
+        }
+    }
+
     /**
      * If we did not find a cached network, build one from the input files. Should throw an exception rather than
      * returning null if for any reason it can't finish building one.
      */
     private @Nonnull TransportNetwork buildNetwork (String networkId) {
         TransportNetwork network;
-        FileStorageKey networkConfigKey = new FileStorageKey(BUNDLES, GTFSCache.cleanId(networkId) + ".json");
-        if (fileStorage.exists(networkConfigKey)) {
-            network = buildNetworkFromConfig(networkId);
-        } else {
-            LOG.warn("Detected old-format bundle stored as single ZIP file");
+        TransportNetworkConfig networkConfig = loadNetworkConfig(networkId);
+        if (networkConfig == null) {
+            // The switch to use JSON manifests instead of zips occurred in 32a1aebe in July 2016.
+            // Over six years have passed, buildNetworkFromBundleZip is deprecated and could probably be removed.
+            LOG.warn("No network config (aka manifest) found. Assuming old-format network inputs bundle stored as a single ZIP file.");
             network = buildNetworkFromBundleZip(networkId);
+        } else {
+            network = buildNetworkFromConfig(networkConfig);
         }
         network.scenarioId = networkId;
 
-        // Networks created in TransportNetworkCache are going to be used for analysis work. Pre-compute distance tables
-        // from stops to street vertices, then pre-build a linked grid pointset for the whole region. These linkages
-        // should be serialized along with the network, which avoids building them when an analysis worker starts.
-        // The linkage we create here will never be used directly, but serves as a basis for scenario linkages, making
-        // analysis much faster to start up.
+        // Pre-compute distance tables from stops out to street vertices, then pre-build a linked grid pointset for the
+        // whole region covered by the street network. These tables and linkages will be serialized along with the
+        // network, which avoids building them when every analysis worker starts. The linkage we create here will never
+        // be used directly, but serves as a basis for scenario linkages, making analyses much faster to start up.
+        // Note, this retains stop-to-vertex distances for the WALK MODE ONLY, even when they are produced as
+        // intermediate results while building linkages for other modes.
+        // This is a candidate for optimization if car or bicycle scenarios are slow to apply.
         network.transitLayer.buildDistanceTables(null);
-        network.rebuildLinkedGridPointSet(StreetMode.WALK);
+
+        Set<StreetMode> buildGridsForModes = Sets.newHashSet(StreetMode.WALK);
+        if (networkConfig != null && networkConfig.buildGridsForModes != null) {
+            buildGridsForModes.addAll(networkConfig.buildGridsForModes);
+        }
+        network.rebuildLinkedGridPointSet(buildGridsForModes);
 
         // Cache the serialized network on the local filesystem and mirror it to any remote storage.
         try {
@@ -247,25 +272,15 @@ private TransportNetwork buildNetworkFromBundleZip (String networkId) {
      * This describes the locations of files used to create a bundle, as well as options applied at network build time.
      * It contains the unique IDs of the GTFS feeds and OSM extract.
      */
-    private TransportNetwork buildNetworkFromConfig (String networkId) {
-        FileStorageKey configFileKey = new FileStorageKey(BUNDLES, getNetworkConfigFilename(networkId));
-        File configFile = fileStorage.getFile(configFileKey);
-        TransportNetworkConfig config;
-
-        try {
-            // Use lenient mapper to mimic behavior in objectFromRequestBody.
-            config = JsonUtilities.lenientObjectMapper.readValue(configFile, TransportNetworkConfig.class);
-        } catch (IOException e) {
-            throw new RuntimeException("Error reading TransportNetworkConfig. Does it contain new unrecognized fields?", e);
-        }
+    private TransportNetwork buildNetworkFromConfig (TransportNetworkConfig config) {
         // FIXME duplicate code. All internal building logic should be encapsulated in a method like
         //  TransportNetwork.build(osm, gtfs1, gtfs2...)
         // We currently have multiple copies of it, in buildNetworkFromConfig and buildNetworkFromBundleZip so you've
         // got to remember to do certain things like set the network ID of the network in multiple places in the code.
         // Maybe we should just completely deprecate bundle ZIPs and remove those code paths.
 
         TransportNetwork network = new TransportNetwork();
-        network.scenarioId = networkId;
+
         network.streetLayer = new StreetLayer();
         network.streetLayer.loadFromOsm(osmCache.get(config.osmId));