From 070050b397ada4deb98a6f5dd1667f2f875e378c Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Wed, 8 Aug 2018 01:38:50 -0700 Subject: [PATCH 01/14] Tile gtfs export --- app/models/gtfs_stop.rb | 1 + app/services/tile_export_service.rb | 106 ++--- app/services/tile_export_service_old.rb | 556 ++++++++++++++++++++++++ 3 files changed, 601 insertions(+), 62 deletions(-) create mode 100644 app/services/tile_export_service_old.rb diff --git a/app/models/gtfs_stop.rb b/app/models/gtfs_stop.rb index 00f7093ab..61f0c8c6d 100644 --- a/app/models/gtfs_stop.rb +++ b/app/models/gtfs_stop.rb @@ -38,6 +38,7 @@ class GTFSStop < ActiveRecord::Base include HasAGeographicGeometry has_many :stop_times, class_name: GTFSStopTime, foreign_key: "stop_id" has_many :gtfs_shapes + has_many :children, class_name: GTFSStop, foreign_key: "parent_station_id" belongs_to :feed_version belongs_to :entity, class_name: 'Stop' validates :feed_version, presence: true, unless: :skip_association_validations diff --git a/app/services/tile_export_service.rb b/app/services/tile_export_service.rb index bc46065bf..b124a5608 100644 --- a/app/services/tile_export_service.rb +++ b/app/services/tile_export_service.rb @@ -92,15 +92,14 @@ def build_stops tileset = TileUtils::TileSet.new(@tilepath) tile = tileset.new_tile(GRAPH_LEVEL, @tile) - Stop - .where_imported_from_feed_version(@feed_version_ids) - .where(parent_stop: nil) + GTFSStop + .where(feed_version_id: @feed_version_ids) + .where(parent_station_id: nil) .geometry_within_bbox(bbox_padded(tile.bbox)) - .includes(:stop_platforms, :stop_egresses) .find_each do |stop| # Check if stop is inside tile - stop_tile = TileUtils::GraphID.new(level: GRAPH_LEVEL, lon: stop.coordinates[0], lat: stop.coordinates[1]).tile + stop_tile = TileUtils::GraphID.new(level: GRAPH_LEVEL, lon: stop.stop_lon, lat: stop.stop_lat).tile if stop_tile != @tile # debug("skipping stop #{stop.id}: coordinates #{stop.coordinates.join(',')} map to tile #{stop_tile} outside of tile #{@tile}") next @@ -110,9 +109,10 @@ def build_stops prev_type_graphid = nil # Egresses - stop_egresses = stop.stop_egresses.to_a - stop_egresses << StopEgress.new(stop.attributes) if stop_egresses.empty? # generated egress + stop_egresses = [] # stop.stop_egresses.to_a + (stop_egresses << GTFSStop.new(stop.attributes)) if stop_egresses.empty? # generated egress stop_egresses.each do |stop_egress| + stop_egress.location_type = 3 node = make_node(stop_egress) node.graphid = TileUtils::GraphID.new(level: GRAPH_LEVEL, tile: @tile, index: tile.message.nodes.size).value node.prev_type_graphid = prev_type_graphid if prev_type_graphid @@ -121,6 +121,7 @@ def build_stops end # Station + stop.location_type = 1 node = make_node(stop) node.graphid = TileUtils::GraphID.new(level: GRAPH_LEVEL, tile: @tile, index: tile.message.nodes.size).value node.prev_type_graphid = prev_type_graphid if prev_type_graphid @@ -128,9 +129,10 @@ def build_stops tile.message.nodes << node # Platforms - stop_platforms = stop.stop_platforms.to_a - (stop_platforms << StopPlatform.new(stop.attributes)) if stop_platforms.empty? # station ssps + stop_platforms = stop.children.to_a + (stop_platforms << GTFSStop.new(stop.attributes)) if stop_platforms.empty? stop_platforms.each do |stop_platform| + stop_platform.location_type = 2 node = make_node(stop_platform) node.graphid = TileUtils::GraphID.new(level: GRAPH_LEVEL, tile: @tile, index: tile.message.nodes.size).value node.prev_type_graphid = prev_type_graphid if prev_type_graphid # station_id graphid @@ -375,36 +377,36 @@ def make_route(route) def make_node(stop) params = {} # float lon = 1; - params[:lon] = stop.coordinates[0] + params[:lon] = stop.stop_lon # float lat = 2; - params[:lat] = stop.coordinates[1] + params[:lat] = stop.stop_lat # uint32 type = 3; - params[:type] = NODE_TYPES[stop.class.name.to_sym] + params[:type] = stop.location_type + 3 # 5 # NODE_TYPES[stop.class.name.to_sym] # uint64 graphid = 4; # set in build_stops # uint64 prev_type_graphid = 5; # set in build_stops # string name = 6; - params[:name] = stop.name + params[:name] = stop.stop_name # string onestop_id = 7; - params[:onestop_id] = stop.onestop_id + params[:onestop_id] = 's-123-test' # uint64 osm_way_id = 8; - params[:osm_way_id] = stop.osm_way_id + params[:osm_way_id] = 123 # stop.osm_way_id # string timezone = 9; - params[:timezone] = stop.timezone + params[:timezone] = stop.stop_timezone # bool wheelchair_boarding = 10; params[:wheelchair_boarding] = true # bool generated = 11; - if stop.instance_of?(StopEgress) && !stop.persisted? - params[:onestop_id] = "#{stop.onestop_id}>" + if stop.location_type == 2 + params[:onestop_id] = "s-123-test>" params[:generated] = true end - if stop.instance_of?(StopPlatform) && !stop.persisted? - params[:onestop_id] = "#{stop.onestop_id}<" + if stop.location_type == 3 + params[:onestop_id] = "s-123-test<" # params[:generated] = true # not set for platforms end # uint32 traversability = 12; - if stop.instance_of?(StopEgress) + if stop.location_type == 3 params[:traversability] = 3 end Valhalla::Mjolnir::Transit::Node.new(params.compact) @@ -459,25 +461,16 @@ def self.export_tiles(tilepath, thread_count: nil, feeds: nil, feed_versions: ni # ActiveRecord::Base.logger = Logger.new(STDOUT) # ActiveRecord::Base.logger.level = Logger::DEBUG - # Avoid autoload issues in threads - Stop.connection - StopPlatform.connection - StopEgress.connection - Route.connection - Operator.connection - RouteStopPattern.connection - EntityImportedFromFeed.connection - ScheduleStopPair.connection - # Filter by feed/feed_version - feed_version_ids = [] - if feed_versions - feed_version_ids = feed_versions.map(&:id) - elsif feeds - feed_version_ids = feeds.map(&:active_feed_version_id) - else - feed_version_ids = Feed.where_active_feed_version_import_level(IMPORT_LEVEL).pluck(:active_feed_version_id) - end + # feed_version_ids = [] + # if feed_versions + # feed_version_ids = feed_versions.map(&:id) + # elsif feeds + # feed_version_ids = feeds.map(&:active_feed_version_id) + # else + # feed_version_ids = Feed.where_active_feed_version_import_level(IMPORT_LEVEL).pluck(:active_feed_version_id) + # end + feed_version_ids = FeedVersion.pluck(:id) # Build bboxes puts "Selecting tiles..." @@ -491,15 +484,13 @@ def self.export_tiles(tilepath, thread_count: nil, feeds: nil, feed_versions: ni FeedVersion.where(id: feed_version_ids).includes(:feed).find_each do |feed_version| feed = feed_version.feed fvtiles = Set.new - Stop.where_imported_from_feed_version(feed_version).find_each do |stop| - if stop.is_a?(StopPlatform) - stop_platforms[stop.parent_stop_id] << stop.id - elsif stop.is_a?(StopEgress) - stop_egresses[stop.parent_stop_id] << stop.id - else - count_stops << stop.id - fvtiles << TileUtils::GraphID.new(level: GRAPH_LEVEL, lon: stop.coordinates[0], lat: stop.coordinates[1]).tile - end + GTFSStop.where(feed_version: feed_version).find_each do |stop| + if stop.parent_station_id.nil? + count_stops << stop.id + fvtiles << TileUtils::GraphID.new(level: GRAPH_LEVEL, lon: stop.stop_lon, lat: stop.stop_lat).tile + else + stop_platforms[stop.parent_stop_id] << stop.id + end end puts "\t(#{count}/#{total}) #{feed_version.feed.onestop_id} #{feed_version.sha1}: #{fvtiles.size} tiles" tiles += fvtiles @@ -518,15 +509,6 @@ def self.export_tiles(tilepath, thread_count: nil, feeds: nil, feed_versions: ni puts "\tnodes: #{count_stops.size + count_egresses + count_platforms}" puts "\tstopid-graphid: #{count_platforms}" - # Clear - count_stops.clear - stop_platforms.clear - stop_egresses.clear - # stopid_graphid = Hash[redis.hgetall('stopid_graphid').map { |k,v| [k.to_i, v.to_i] }] - # expected_stops = Set.new - # count_stops.each { |i| expected_stops += (stop_platforms[i].empty? ? [i].to_set : stop_platforms[i]) } - # missing = stopid_graphid.keys.to_set - expected_stops - # Setup queue thread_count ||= 1 redis = Redis.new @@ -545,11 +527,11 @@ def self.export_tiles(tilepath, thread_count: nil, feeds: nil, feed_versions: ni puts "\nStops finished. Schedule tile queue: #{redis.llen(KEY_QUEUE_SCHEDULES)} stopid-graphid mappings: #{redis.hlen(KEY_STOPID_GRAPHID)}" # Build schedule, routes, shapes for each tile. - puts "\n===== Routes, Shapes, StopPairs =====\n" - workers = (0...thread_count).map do - fork { tile_build_schedules(tilepath, feed_version_ids: feed_version_ids) } - end - workers.each { |pid| Process.wait(pid) } + # puts "\n===== Routes, Shapes, StopPairs =====\n" + # workers = (0...thread_count).map do + # fork { tile_build_schedules(tilepath, feed_version_ids: feed_version_ids) } + # end + # workers.each { |pid| Process.wait(pid) } puts "Done!" end diff --git a/app/services/tile_export_service_old.rb b/app/services/tile_export_service_old.rb new file mode 100644 index 000000000..2a548a184 --- /dev/null +++ b/app/services/tile_export_service_old.rb @@ -0,0 +1,556 @@ +module TileExportService + BBOX_PADDING = 0.1 + KEY_QUEUE_STOPS = 'queue_stops' + KEY_QUEUE_SCHEDULES = 'queue_schedules' + KEY_STOPID_GRAPHID = 'stopid_graphid' + IMPORT_LEVEL = 4 + GRAPH_LEVEL = 2 + STOP_PAIRS_TILE_LIMIT = 500_000 + + # kTransitEgress = 4, // Transit egress + # kTransitStation = 5, // Transit station + # kMultiUseTransitPlatform = 6, // Multi-use transit platform (rail and bus) + NODE_TYPES = { + StopEgress: 4, + Stop: 5, + StopPlatform: 6 + } + + VT = Valhalla::Mjolnir::Transit::VehicleType + VEHICLE_TYPES = { + tram: VT::Tram, + tram_service: VT::Tram, + metro: VT::Metro, + rail: VT::Rail, + suburban_railway: VT::Rail, + bus: VT::Bus, + trolleybys_service: VT::Bus, + express_bus_service: VT::Bus, + local_bus_service: VT::Bus, + bus_service: VT::Bus, + shuttle_bus: VT::Bus, + demand_and_response_bus_service: VT::Bus, + regional_bus_service: VT::Bus, + ferry: VT::Ferry, + cablecar: VT::CableCar, + gondola: VT::Gondola, + funicular: VT::Funicular + } + + class TileValueError < StandardError + end + + class OriginEqualsDestinationError < TileValueError + end + + class MissingGraphIDError < TileValueError + end + + class MissingRouteError < TileValueError + end + + class MissingShapeError < TileValueError + end + + class MissingTripError < TileValueError + end + + class InvalidTimeError < TileValueError + end + + class TileBuilder + attr_accessor :tile + def initialize(tilepath, tile, feed_version_ids: nil) + @tilepath = tilepath + @tile = tile + # filters + @feed_version_ids = feed_version_ids || [] + # globally unique indexes + @stopid_graphid ||= {} + @graphid_stopid ||= {} + @trip_index ||= TileUtils::DigestIndex.new(bits: 24) + @block_index ||= TileUtils::DigestIndex.new(start: 1, bits: 20) + # tile unique indexes + @route_index = {} + @shape_index = {} + end + + def log(msg) + puts "tile #{@tile}: #{msg}" + end + + def debug(msg) + puts "tile #{@tile} debug: #{msg}" + end + + def build_stops + # TODO: + # max graph_ids in a tile + t = Time.now + + # New tile + tileset = TileUtils::TileSet.new(@tilepath) + tile = tileset.new_tile(GRAPH_LEVEL, @tile) + + Stop + .where_imported_from_feed_version(@feed_version_ids) + .where(parent_stop: nil) + .geometry_within_bbox(bbox_padded(tile.bbox)) + .includes(:stop_platforms, :stop_egresses) + .find_each do |stop| + + # Check if stop is inside tile + stop_tile = TileUtils::GraphID.new(level: GRAPH_LEVEL, lon: stop.coordinates[0], lat: stop.coordinates[1]).tile + if stop_tile != @tile + # debug("skipping stop #{stop.id}: coordinates #{stop.coordinates.join(',')} map to tile #{stop_tile} outside of tile #{@tile}") + next + end + + # Station references + prev_type_graphid = nil + + # Egresses + stop_egresses = stop.stop_egresses.to_a + stop_egresses << StopEgress.new(stop.attributes) if stop_egresses.empty? # generated egress + stop_egresses.each do |stop_egress| + node = make_node(stop_egress) + node.graphid = TileUtils::GraphID.new(level: GRAPH_LEVEL, tile: @tile, index: tile.message.nodes.size).value + node.prev_type_graphid = prev_type_graphid if prev_type_graphid + prev_type_graphid ||= node.graphid + tile.message.nodes << node + end + + # Station + node = make_node(stop) + node.graphid = TileUtils::GraphID.new(level: GRAPH_LEVEL, tile: @tile, index: tile.message.nodes.size).value + node.prev_type_graphid = prev_type_graphid if prev_type_graphid + prev_type_graphid = node.graphid + tile.message.nodes << node + + # Platforms + stop_platforms = stop.stop_platforms.to_a + (stop_platforms << StopPlatform.new(stop.attributes)) if stop_platforms.empty? # station ssps + stop_platforms.each do |stop_platform| + node = make_node(stop_platform) + node.graphid = TileUtils::GraphID.new(level: GRAPH_LEVEL, tile: @tile, index: tile.message.nodes.size).value + node.prev_type_graphid = prev_type_graphid if prev_type_graphid # station_id graphid + @stopid_graphid[stop_platform.id] = node.graphid + @graphid_stopid[node.graphid] = stop_platform.id + tile.message.nodes << node + end + end + + # Write tile + nodes_size = tile.message.nodes.size + tileset.write_tile(tile) if nodes_size > 0 + t = Time.now - t + log("nodes: #{nodes_size} time: #{t.round(2)} (#{(nodes_size/t).to_i} nodes/s)") + return nodes_size + end + + def build_schedules + # Get stop_ids + t = Time.now + tileset = TileUtils::TileSet.new(@tilepath) + base_tile = tileset.read_tile(GRAPH_LEVEL, @tile) + stop_ids = base_tile.message.nodes.map { |node| @graphid_stopid[node.graphid] }.compact + + # Build stop_pairs for each stop_id + tile_ext = 0 + stop_pairs_tile = base_tile # tileset.new_tile(GRAPH_LEVEL, @tile) + stop_pairs_total = 0 + errors = Hash.new(0) + stop_ids.each do |stop_id| + stop_pairs_stop_id_count = 0 + ScheduleStopPair + .where(origin_id: stop_id) + .includes(:origin, :destination, :operator) + .find_in_batches do |ssps| + # Evaluate by active feed version - faster than join or where in (?) + ssps = ssps.select { |ssp| @feed_version_ids.include?(ssp.feed_version_id) } + + # Get unvisited routes for this batch + route_ids = ssps.map(&:route_id).select { |route_id| !@route_index.key?(route_id) } + ssps.each { |ssp| @route_index[ssp.route_id] ||= nil } # set as visited + Route.where(id: route_ids).find_each do |route| # get unvisited + @route_index[route.id] = base_tile.message.routes.size + debug("route: #{route.id} -> #{@route_index[route.id]}") + base_tile.message.routes << make_route(route) + end + + # Get unseen rsps for this batch + rsp_ids = ssps.map(&:route_stop_pattern_id).select { |rsp_id| !@shape_index.key?(rsp_id) } + ssps.each { |ssp| @shape_index[ssp.route_stop_pattern_id] ||= nil } # set as visited + RouteStopPattern.where(id: rsp_ids).find_each do |rsp| + shape = make_shape(rsp) + shape.shape_id = base_tile.message.shapes.size + 1 + @shape_index[rsp.id] = shape.shape_id + debug("shape: #{rsp.id} -> #{@shape_index[rsp.id]}") + base_tile.message.shapes << shape + end + + # Process each ssp + ssps.each do |ssp| + # process ssp and count errors + begin + stop_pairs_tile.message.stop_pairs << make_stop_pair(ssp) + stop_pairs_stop_id_count += 1 + stop_pairs_total += 1 + rescue TileValueError => e + errors[e.class.name.to_sym] += 1 + log("error: ssp #{ssp.id}: #{e}") + rescue StandardError => e + errors[e.class.name.to_sym] += 1 + log("error: ssp #{ssp.id}: #{e}") + end + end + + # Write supplement tile, start new tile + if stop_pairs_tile.message.stop_pairs.size > STOP_PAIRS_TILE_LIMIT + if stop_pairs_tile != base_tile + debug("writing tile ext #{tile_ext}: #{stop_pairs_tile.message.stop_pairs.size} stop_pairs") + tileset.write_tile(stop_pairs_tile, ext: tile_ext) + tile_ext += 1 + end + stop_pairs_tile = tileset.new_tile(GRAPH_LEVEL, @tile) + end + end + # Done for this stop + debug("stop_pairs for stop_id #{stop_id}: #{stop_pairs_stop_id_count}") + end + + # Write dangling supplement tile + if stop_pairs_tile != base_tile && stop_pairs_tile.message.stop_pairs.size > 0 + debug("writing tile ext #{tile_ext}: #{stop_pairs_tile.message.stop_pairs.size} stop_pairs") + tileset.write_tile(stop_pairs_tile, ext: tile_ext) + end + + # Write the base tile + debug("writing tile base: #{base_tile.message.nodes.size} nodes, #{base_tile.message.routes.size} routes, #{base_tile.message.shapes.size} shapes, #{base_tile.message.stop_pairs.size} stop_pairs (#{stop_pairs_total} tile total)") + tileset.write_tile(base_tile) + + # Write tile + t = Time.now - t + error_txt = ([errors.values.sum.to_s] + errors.map { |k,v| "#{k}: #{v}" }).join(' ') + log("#{base_tile.message.nodes.size} nodes, #{base_tile.message.routes.size} routes, #{base_tile.message.shapes.size} shapes, #{stop_pairs_total} stop_pairs, errors #{error_txt}, time: #{t.round(2)} (#{(stop_pairs_total/t).to_i} stop_pairs/s)") + return stop_pairs_total + end + + private + + def seconds_since_midnight(value) + h,m,s = value.split(':').map(&:to_i) + h * 3600 + m * 60 + s + end + + def color_to_int(value) + match = /(\h{6})/.match(value.to_s) + match ? match[0].to_i(16) : nil + end + + # bbox padding + def bbox_padded(bbox) + ymin, xmin, ymax, xmax = bbox + padding = BBOX_PADDING + [ymin-padding, xmin-padding, ymax+padding, xmax+padding] + end + + # make entity methods + def make_stop_pair(ssp) + # TODO: + # skip if origin_departure_time < frequency_start_time + # skip if bad time information + # add < and > to onestop_ids + destination_graphid = @stopid_graphid[ssp.destination_id] + origin_graphid = @stopid_graphid[ssp.origin_id] + fail OriginEqualsDestinationError.new("origin_graphid #{origin_graphid} == destination_graphid #{destination_graphid}") if origin_graphid == destination_graphid + fail MissingGraphIDError.new("missing origin_graphid for stop #{ssp.origin_id}") unless origin_graphid + fail MissingGraphIDError.new("missing destination_graphid for stop #{ssp.destination_id}") unless destination_graphid + + route_index = @route_index[ssp.route_id] + fail MissingRouteError.new("missing route_index for route #{ssp.route_id}") unless route_index + + shape_id = @shape_index[ssp.route_stop_pattern_id] + fail MissingShapeError.new("missing shape for rsp #{ssp.route_stop_pattern_id}") unless shape_id + + trip_id = @trip_index.check(ssp.trip) + fail MissingTripError.new("missing trip_id for trip #{ssp.trip}") unless trip_id + + destination_arrival_time = seconds_since_midnight(ssp.destination_arrival_time) + origin_departure_time = seconds_since_midnight(ssp.origin_departure_time) + fail InvalidTimeError.new("origin_departure_time #{origin_departure_time} > destination_arrival_time #{destination_arrival_time}") if origin_departure_time > destination_arrival_time + + block_id = @block_index.check(ssp.block_id) + + # Make SSP + params = {} + # bool bikes_allowed = 1; + # uint32 block_id = 2; + # params[:block_id] = block_id + # uint32 destination_arrival_time = 3; + params[:destination_arrival_time] = destination_arrival_time + # uint64 destination_graphid = 4; + params[:destination_graphid] = destination_graphid + # string destination_onestop_id = 5; + params[:destination_onestop_id] = ssp.destination.onestop_id + # string operated_by_onestop_id = 6; + params[:operated_by_onestop_id] = ssp.operator.onestop_id + # uint32 origin_departure_time = 7; + params[:origin_departure_time] = origin_departure_time + # uint64 origin_graphid = 8; + params[:origin_graphid] = origin_graphid + # string origin_onestop_id = 9; + params[:origin_onestop_id] = ssp.origin.onestop_id + # uint32 route_index = 10; + params[:route_index] = route_index + # repeated uint32 service_added_dates = 11; + params[:service_added_dates] = ssp.service_added_dates.map(&:jd) + # repeated bool service_days_of_week = 12; + params[:service_days_of_week] = ssp.service_days_of_week + # uint32 service_end_date = 13; + params[:service_end_date] = ssp.service_end_date.jd + # repeated uint32 service_except_dates = 14; + params[:service_except_dates] = ssp.service_except_dates.map(&:jd) + # uint32 service_start_date = 15; + params[:service_start_date] = ssp.service_start_date.jd + # string trip_headsign = 16; + params[:trip_headsign] = ssp.trip_headsign + # uint32 trip_id = 17; + params[:trip_id] = trip_id + # bool wheelchair_accessible = 18; + params[:wheelchair_accessible] = true # !!(ssp.wheelchair_accessible) + # uint32 shape_id = 20; + params[:shape_id] = shape_id + # float origin_dist_traveled = 21; + params[:origin_dist_traveled] = ssp.origin_dist_traveled if ssp.origin_dist_traveled + # float destination_dist_traveled = 22; + params[:destination_dist_traveled] = ssp.destination_dist_traveled if ssp.destination_dist_traveled + if ssp.frequency_headway_seconds + # protobuf doesn't define frequency_start_time + # uint32 frequency_end_time = 23; + params[:frequency_end_time] = seconds_since_midnight(ssp.frequency_end_time) + # uint32 frequency_headway_seconds = 24; + params[:frequency_headway_seconds] = ssp.frequency_headway_seconds + end + Valhalla::Mjolnir::Transit::StopPair.new(params) + end + + def make_shape(rsp) + params = {} + # uint32 shape_id = 1; + # bytes encoded_shape = 2; + # reverse coordinates + reversed = rsp.geometry[:coordinates].map { |a,b| [b,a] } + params[:encoded_shape] = TileUtils::Shape7.encode(reversed) + Valhalla::Mjolnir::Transit::Shape.new(params) + end + + def make_route(route) + # TODO: + # skip if unknown vehicle_type + params = {} + # string name = 1; + params[:name] = route.name + # string onestop_id = 2; + params[:onestop_id] = route.onestop_id + # string operated_by_name = 3; + params[:operated_by_name] = route.operator.name + # string operated_by_onestop_id = 4; + params[:operated_by_onestop_id] = route.operator.onestop_id + # string operated_by_website = 5; + params[:operated_by_website] = route.operator.website + # uint32 route_color = 6; + params[:route_color] = color_to_int(route.color || 'FFFFFF') + # string route_desc = 7; + params[:route_desc] = route.tags["route_desc"] + # string route_long_name = 8; + params[:route_long_name] = route.tags["route_long_name"] || route.name + # uint32 route_text_color = 9; + params[:route_text_color] = color_to_int(route.tags["route_text_color"]) + # VehicleType vehicle_type = 10; + params[:vehicle_type] = VEHICLE_TYPES[route.vehicle_type.to_sym] || VT::Bus + Valhalla::Mjolnir::Transit::Route.new(params.compact) + end + + def make_node(stop) + params = {} + # float lon = 1; + params[:lon] = stop.coordinates[0] + # float lat = 2; + params[:lat] = stop.coordinates[1] + # uint32 type = 3; + params[:type] = NODE_TYPES[stop.class.name.to_sym] + # uint64 graphid = 4; + # set in build_stops + # uint64 prev_type_graphid = 5; + # set in build_stops + # string name = 6; + params[:name] = stop.name + # string onestop_id = 7; + params[:onestop_id] = stop.onestop_id + # uint64 osm_way_id = 8; + params[:osm_way_id] = stop.osm_way_id + # string timezone = 9; + params[:timezone] = stop.timezone + # bool wheelchair_boarding = 10; + params[:wheelchair_boarding] = true + # bool generated = 11; + if stop.instance_of?(StopEgress) && !stop.persisted? + params[:onestop_id] = "#{stop.onestop_id}>" + params[:generated] = true + end + if stop.instance_of?(StopPlatform) && !stop.persisted? + params[:onestop_id] = "#{stop.onestop_id}<" + # params[:generated] = true # not set for platforms + end + # uint32 traversability = 12; + if stop.instance_of?(StopEgress) + params[:traversability] = 3 + end + Valhalla::Mjolnir::Transit::Node.new(params.compact) + end + end + + def self.tile_build_stops(tilepath, feed_version_ids: nil) + redis = Redis.new + while tile = redis.rpop(KEY_QUEUE_STOPS) + tile = tile.to_i + builder = TileBuilder.new(tilepath, tile, feed_version_ids: feed_version_ids) + nodes_size = builder.build_stops + if nodes_size > 0 + redis.rpush(KEY_QUEUE_SCHEDULES, tile) + stopid_graphid = builder.instance_variable_get('@stopid_graphid') + stopid_graphid.each_slice(1000) { |i| redis.hmset(KEY_STOPID_GRAPHID, i.flatten) } + end + remaining = redis.llen(KEY_QUEUE_STOPS) + puts "remaining: ~#{remaining}" + end + end + + def self.tile_build_schedules(tilepath, feed_version_ids: nil) + # stopid_graphid + stopid_graphid = {} + graphid_stopid = {} + redis = Redis.new + cursor = nil + while cursor != '0' + cursor, data = redis.hscan(KEY_STOPID_GRAPHID, cursor, count: 1_000) + data.each do |k,v| + k = k.to_i + v = v.to_i + stopid_graphid[k] = v + graphid_stopid[v] = k + end + end + # queue + while tile = redis.rpop(KEY_QUEUE_SCHEDULES) + tile = tile.to_i + builder = TileBuilder.new(tilepath, tile, feed_version_ids: feed_version_ids) + builder.instance_variable_set('@stopid_graphid', stopid_graphid) + builder.instance_variable_set('@graphid_stopid', graphid_stopid) + builder.build_schedules + remaining = redis.llen(KEY_QUEUE_SCHEDULES) + puts "remaining: ~#{remaining}" + end + end + + def self.export_tiles(tilepath, thread_count: nil, feeds: nil, feed_versions: nil, tiles: nil) + # Debug + # ActiveRecord::Base.logger = Logger.new(STDOUT) + # ActiveRecord::Base.logger.level = Logger::DEBUG + + # Avoid autoload issues in threads + Stop.connection + StopPlatform.connection + StopEgress.connection + Route.connection + Operator.connection + RouteStopPattern.connection + EntityImportedFromFeed.connection + ScheduleStopPair.connection + + # Filter by feed/feed_version + feed_version_ids = [] + if feed_versions + feed_version_ids = feed_versions.map(&:id) + elsif feeds + feed_version_ids = feeds.map(&:active_feed_version_id) + else + feed_version_ids = Feed.where_active_feed_version_import_level(IMPORT_LEVEL).pluck(:active_feed_version_id) + end + + # Build bboxes + puts "Selecting tiles..." + count_stops = Set.new + stop_platforms = Hash.new { |h,k| h[k] = Set.new } + stop_egresses = Hash.new { |h,k| h[k] = Set.new } + tiles = Set.new(tiles) + if tiles.empty? + count = 1 + total = feed_version_ids.size + FeedVersion.where(id: feed_version_ids).includes(:feed).find_each do |feed_version| + feed = feed_version.feed + fvtiles = Set.new + Stop.where_imported_from_feed_version(feed_version).find_each do |stop| + if stop.is_a?(StopPlatform) + stop_platforms[stop.parent_stop_id] << stop.id + elsif stop.is_a?(StopEgress) + stop_egresses[stop.parent_stop_id] << stop.id + else + count_stops << stop.id + fvtiles << TileUtils::GraphID.new(level: GRAPH_LEVEL, lon: stop.coordinates[0], lat: stop.coordinates[1]).tile + end + end + puts "\t(#{count}/#{total}) #{feed_version.feed.onestop_id} #{feed_version.sha1}: #{fvtiles.size} tiles" + tiles += fvtiles + count += 1 + end + end + + # TODO: Filter stop_platforms/stop_egresses by feed_version + count_egresses = count_stops.map { |i| stop_egresses[i].empty? ? 1 : stop_egresses[i].size }.sum + count_platforms = count_stops.map { |i| stop_platforms[i].empty? ? 1 : stop_platforms[i].size }.sum + puts "Tiles to build: #{tiles.size}" + puts "Expected:" + puts "\tstops: #{count_stops.size}" + puts "\tplatforms: #{stop_platforms.map { |k,v| v.size }.sum}" + puts "\tegresses: #{stop_egresses.map { |k,v| v.size }.sum}" + puts "\tnodes: #{count_stops.size + count_egresses + count_platforms}" + puts "\tstopid-graphid: #{count_platforms}" + + # Clear + count_stops.clear + stop_platforms.clear + stop_egresses.clear + # stopid_graphid = Hash[redis.hgetall('stopid_graphid').map { |k,v| [k.to_i, v.to_i] }] + # expected_stops = Set.new + # count_stops.each { |i| expected_stops += (stop_platforms[i].empty? ? [i].to_set : stop_platforms[i]) } + # missing = stopid_graphid.keys.to_set - expected_stops + + # Setup queue + thread_count ||= 1 + redis = Redis.new + redis.del(KEY_QUEUE_STOPS) + redis.del(KEY_QUEUE_SCHEDULES) + redis.del(KEY_STOPID_GRAPHID) + tiles.each_slice(1000) { |i| redis.rpush(KEY_QUEUE_STOPS, i) } + + # Build stops for each tile. + puts "\n===== Stops =====\n" + workers = (0...thread_count).map do + fork { tile_build_stops(tilepath, feed_version_ids: feed_version_ids) } + end + workers.each { |pid| Process.wait(pid) } + + puts "\nStops finished. Schedule tile queue: #{redis.llen(KEY_QUEUE_SCHEDULES)} stopid-graphid mappings: #{redis.hlen(KEY_STOPID_GRAPHID)}" + + # Build schedule, routes, shapes for each tile. + puts "\n===== Routes, Shapes, StopPairs =====\n" + workers = (0...thread_count).map do + fork { tile_build_schedules(tilepath, feed_version_ids: feed_version_ids) } + end + workers.each { |pid| Process.wait(pid) } + + puts "Done!" + end +end \ No newline at end of file From 32c5c0546151b9f7d6b1beab23dbec66ad0b8e99 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Wed, 8 Aug 2018 02:13:22 -0700 Subject: [PATCH 02/14] Updating to GTFS models --- app/models/gtfs_route.rb | 5 ++ app/services/tile_export_service.rb | 100 +++++++++++++++------------- 2 files changed, 58 insertions(+), 47 deletions(-) diff --git a/app/models/gtfs_route.rb b/app/models/gtfs_route.rb index daee5bd6b..7e9a4006d 100644 --- a/app/models/gtfs_route.rb +++ b/app/models/gtfs_route.rb @@ -46,4 +46,9 @@ class GTFSRoute < ActiveRecord::Base validates :route_id, presence: true validates :route_type, presence: true validate { errors.add("route_short_name or route_long_name must be present") unless route_short_name.presence || route_long_name.presence } + + def name + route_short_name.presence || route_long_name.presence + end + end diff --git a/app/services/tile_export_service.rb b/app/services/tile_export_service.rb index b124a5608..dd658bac3 100644 --- a/app/services/tile_export_service.rb +++ b/app/services/tile_export_service.rb @@ -11,9 +11,9 @@ module TileExportService # kTransitStation = 5, // Transit station # kMultiUseTransitPlatform = 6, // Multi-use transit platform (rail and bus) NODE_TYPES = { - StopEgress: 4, - Stop: 5, - StopPlatform: 6 + 2 => 4, + 1 => 5, + 0 => 6 } VT = Valhalla::Mjolnir::Transit::VehicleType @@ -112,7 +112,7 @@ def build_stops stop_egresses = [] # stop.stop_egresses.to_a (stop_egresses << GTFSStop.new(stop.attributes)) if stop_egresses.empty? # generated egress stop_egresses.each do |stop_egress| - stop_egress.location_type = 3 + stop_egress.location_type = 2 node = make_node(stop_egress) node.graphid = TileUtils::GraphID.new(level: GRAPH_LEVEL, tile: @tile, index: tile.message.nodes.size).value node.prev_type_graphid = prev_type_graphid if prev_type_graphid @@ -132,7 +132,7 @@ def build_stops stop_platforms = stop.children.to_a (stop_platforms << GTFSStop.new(stop.attributes)) if stop_platforms.empty? stop_platforms.each do |stop_platform| - stop_platform.location_type = 2 + stop_platform.location_type = 0 node = make_node(stop_platform) node.graphid = TileUtils::GraphID.new(level: GRAPH_LEVEL, tile: @tile, index: tile.message.nodes.size).value node.prev_type_graphid = prev_type_graphid if prev_type_graphid # station_id graphid @@ -163,27 +163,28 @@ def build_schedules stop_pairs_total = 0 errors = Hash.new(0) stop_ids.each do |stop_id| + puts "stop_id: #{stop_id}" stop_pairs_stop_id_count = 0 - ScheduleStopPair - .where(origin_id: stop_id) - .includes(:origin, :destination, :operator) + GTFSStopTime + .where(stop_id: stop_id) + .includes(:trip) .find_in_batches do |ssps| # Evaluate by active feed version - faster than join or where in (?) ssps = ssps.select { |ssp| @feed_version_ids.include?(ssp.feed_version_id) } # Get unvisited routes for this batch - route_ids = ssps.map(&:route_id).select { |route_id| !@route_index.key?(route_id) } - ssps.each { |ssp| @route_index[ssp.route_id] ||= nil } # set as visited - Route.where(id: route_ids).find_each do |route| # get unvisited + route_ids = ssps.map { |ssp| ssp.trip.route_id }.select { |route_id| !@route_index.key?(route_id) } + ssps.each { |ssp| @route_index[ssp.trip.route_id] ||= nil } # set as visited + GTFSRoute.where(id: route_ids).find_each do |route| # get unvisited @route_index[route.id] = base_tile.message.routes.size debug("route: #{route.id} -> #{@route_index[route.id]}") base_tile.message.routes << make_route(route) end # Get unseen rsps for this batch - rsp_ids = ssps.map(&:route_stop_pattern_id).select { |rsp_id| !@shape_index.key?(rsp_id) } - ssps.each { |ssp| @shape_index[ssp.route_stop_pattern_id] ||= nil } # set as visited - RouteStopPattern.where(id: rsp_ids).find_each do |rsp| + rsp_ids = ssps.map { |ssp| ssp.trip.shape_id }.select { |rsp_id| !@shape_index.key?(rsp_id) } + ssps.each { |ssp| @shape_index[ssp.trip.shape_id] ||= nil } # set as visited + GTFSShape.where(id: rsp_ids).find_each do |rsp| shape = make_shape(rsp) shape.shape_id = base_tile.message.shapes.size + 1 @shape_index[rsp.id] = shape.shape_id @@ -259,27 +260,30 @@ def bbox_padded(bbox) # make entity methods def make_stop_pair(ssp) + params = {} + return Valhalla::Mjolnir::Transit::StopPair.new(params) + # TODO: # skip if origin_departure_time < frequency_start_time # skip if bad time information # add < and > to onestop_ids destination_graphid = @stopid_graphid[ssp.destination_id] - origin_graphid = @stopid_graphid[ssp.origin_id] + origin_graphid = @stopid_graphid[ssp.stop_id] fail OriginEqualsDestinationError.new("origin_graphid #{origin_graphid} == destination_graphid #{destination_graphid}") if origin_graphid == destination_graphid - fail MissingGraphIDError.new("missing origin_graphid for stop #{ssp.origin_id}") unless origin_graphid + fail MissingGraphIDError.new("missing origin_graphid for stop #{ssp.stop_id}") unless origin_graphid fail MissingGraphIDError.new("missing destination_graphid for stop #{ssp.destination_id}") unless destination_graphid - route_index = @route_index[ssp.route_id] - fail MissingRouteError.new("missing route_index for route #{ssp.route_id}") unless route_index + route_index = @route_index[ssp.trip.route_id] + fail MissingRouteError.new("missing route_index for route #{ssp.trip.route_id}") unless route_index - shape_id = @shape_index[ssp.route_stop_pattern_id] - fail MissingShapeError.new("missing shape for rsp #{ssp.route_stop_pattern_id}") unless shape_id + shape_id = @shape_index[ssp.trip.shape_id] + fail MissingShapeError.new("missing shape for rsp #{ssp.trip.shape_id}") unless shape_id - trip_id = @trip_index.check(ssp.trip) - fail MissingTripError.new("missing trip_id for trip #{ssp.trip}") unless trip_id + trip_id = @trip_index.check(ssp.trip.trip_id) + fail MissingTripError.new("missing trip_id for trip #{ssp.trip.trip_id}") unless trip_id destination_arrival_time = seconds_since_midnight(ssp.destination_arrival_time) - origin_departure_time = seconds_since_midnight(ssp.origin_departure_time) + origin_departure_time = seconds_since_midnight(ssp.departure_time) fail InvalidTimeError.new("origin_departure_time #{origin_departure_time} > destination_arrival_time #{destination_arrival_time}") if origin_departure_time > destination_arrival_time block_id = @block_index.check(ssp.block_id) @@ -294,15 +298,15 @@ def make_stop_pair(ssp) # uint64 destination_graphid = 4; params[:destination_graphid] = destination_graphid # string destination_onestop_id = 5; - params[:destination_onestop_id] = ssp.destination.onestop_id + params[:destination_onestop_id] = 's-123-destination' # string operated_by_onestop_id = 6; - params[:operated_by_onestop_id] = ssp.operator.onestop_id + params[:operated_by_onestop_id] = 'o-123-test' # uint32 origin_departure_time = 7; params[:origin_departure_time] = origin_departure_time # uint64 origin_graphid = 8; params[:origin_graphid] = origin_graphid # string origin_onestop_id = 9; - params[:origin_onestop_id] = ssp.origin.onestop_id + params[:origin_onestop_id] = 's-123-origin' # uint32 route_index = 10; params[:route_index] = route_index # repeated uint32 service_added_dates = 11; @@ -354,34 +358,36 @@ def make_route(route) # string name = 1; params[:name] = route.name # string onestop_id = 2; - params[:onestop_id] = route.onestop_id + params[:onestop_id] = 'r-123-test' # string operated_by_name = 3; - params[:operated_by_name] = route.operator.name + params[:operated_by_name] = route.agency.agency_name # string operated_by_onestop_id = 4; - params[:operated_by_onestop_id] = route.operator.onestop_id + params[:operated_by_onestop_id] = 'o-123-test' # string operated_by_website = 5; - params[:operated_by_website] = route.operator.website + params[:operated_by_website] = route.agency.agency_url # uint32 route_color = 6; - params[:route_color] = color_to_int(route.color || 'FFFFFF') + params[:route_color] = color_to_int(route.route_color || 'FFFFFF') # string route_desc = 7; - params[:route_desc] = route.tags["route_desc"] + params[:route_desc] = route.route_desc # string route_long_name = 8; - params[:route_long_name] = route.tags["route_long_name"] || route.name + params[:route_long_name] = route.route_long_name # uint32 route_text_color = 9; - params[:route_text_color] = color_to_int(route.tags["route_text_color"]) + params[:route_text_color] = color_to_int(route.route_text_color) # VehicleType vehicle_type = 10; - params[:vehicle_type] = VEHICLE_TYPES[route.vehicle_type.to_sym] || VT::Bus + params[:vehicle_type] = VT::Bus Valhalla::Mjolnir::Transit::Route.new(params.compact) end def make_node(stop) + onestop_id = 's-123-test' + osm_way_id = 123 params = {} # float lon = 1; params[:lon] = stop.stop_lon # float lat = 2; params[:lat] = stop.stop_lat # uint32 type = 3; - params[:type] = stop.location_type + 3 # 5 # NODE_TYPES[stop.class.name.to_sym] + params[:type] = NODE_TYPES[stop.location_type] # uint64 graphid = 4; # set in build_stops # uint64 prev_type_graphid = 5; @@ -389,24 +395,24 @@ def make_node(stop) # string name = 6; params[:name] = stop.stop_name # string onestop_id = 7; - params[:onestop_id] = 's-123-test' + params[:onestop_id] = onestop_id # uint64 osm_way_id = 8; - params[:osm_way_id] = 123 # stop.osm_way_id + params[:osm_way_id] = osm_way_id # string timezone = 9; params[:timezone] = stop.stop_timezone # bool wheelchair_boarding = 10; params[:wheelchair_boarding] = true # bool generated = 11; if stop.location_type == 2 - params[:onestop_id] = "s-123-test>" + params[:onestop_id] = "#{onestop_id}>" params[:generated] = true end - if stop.location_type == 3 - params[:onestop_id] = "s-123-test<" + if stop.location_type == 0 + params[:onestop_id] = "#{onestop_id}<" # params[:generated] = true # not set for platforms end # uint32 traversability = 12; - if stop.location_type == 3 + if stop.location_type == 2 params[:traversability] = 3 end Valhalla::Mjolnir::Transit::Node.new(params.compact) @@ -527,11 +533,11 @@ def self.export_tiles(tilepath, thread_count: nil, feeds: nil, feed_versions: ni puts "\nStops finished. Schedule tile queue: #{redis.llen(KEY_QUEUE_SCHEDULES)} stopid-graphid mappings: #{redis.hlen(KEY_STOPID_GRAPHID)}" # Build schedule, routes, shapes for each tile. - # puts "\n===== Routes, Shapes, StopPairs =====\n" - # workers = (0...thread_count).map do - # fork { tile_build_schedules(tilepath, feed_version_ids: feed_version_ids) } - # end - # workers.each { |pid| Process.wait(pid) } + puts "\n===== Routes, Shapes, StopPairs =====\n" + workers = (0...thread_count).map do + fork { tile_build_schedules(tilepath, feed_version_ids: feed_version_ids) } + end + workers.each { |pid| Process.wait(pid) } puts "Done!" end From 88b098e15b47d17b2f78300e0a986e69b16a5499 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Wed, 8 Aug 2018 17:55:55 -0700 Subject: [PATCH 03/14] Fix interpolation bug --- app/services/gtfs_import_service.rb | 34 +++---- app/services/tile_export_service.rb | 117 ++++++++++++------------ app/services/tile_export_service_old.rb | 2 +- 3 files changed, 78 insertions(+), 75 deletions(-) diff --git a/app/services/gtfs_import_service.rb b/app/services/gtfs_import_service.rb index 5d05b613a..830150f9a 100644 --- a/app/services/gtfs_import_service.rb +++ b/app/services/gtfs_import_service.rb @@ -344,25 +344,20 @@ def import_trip(trip, stop_times) log("processing trip: #{trip.id} stop_times #{stop_times.size}") # Create stop_times trip_stop_times = [] - stop_times.each_index do |i| - origin = stop_times[i] - destination = stop_times[i+1] # last stop is nil - next unless @stop_ids[origin.stop_id] && (destination.nil? || @stop_ids[destination.stop_id]) + stop_times.each do |stop_time| + next unless @stop_ids[stop_time.stop_id] params = {} params[:feed_version_id] = @feed_version.id - params[:stop_sequence] = gtfs_int(origin.stop_sequence) - params[:stop_headsign] = origin.stop_headsign - params[:pickup_type] = gtfs_int(origin.pickup_type) || 0 - params[:drop_off_type] = gtfs_int(origin.drop_off_type) || 0 - params[:shape_dist_traveled] = gtfs_float(origin.shape_dist_traveled) - params[:timepoint] = gtfs_int(origin.timepoint) + params[:stop_sequence] = gtfs_int(stop_time.stop_sequence) + params[:stop_headsign] = stop_time.stop_headsign + params[:pickup_type] = gtfs_int(stop_time.pickup_type) || 0 + params[:drop_off_type] = gtfs_int(stop_time.drop_off_type) || 0 + params[:shape_dist_traveled] = gtfs_float(stop_time.shape_dist_traveled) + params[:timepoint] = gtfs_int(stop_time.timepoint) # where - params[:stop_id] = @stop_ids[origin.stop_id] - params[:arrival_time] = gtfs_time(origin.arrival_time) - params[:departure_time] = gtfs_time(origin.departure_time) - # for convenience - params[:destination_id] = @stop_ids[destination.stop_id] if destination - params[:destination_arrival_time] = gtfs_time(destination.arrival_time) if destination + params[:stop_id] = @stop_ids[stop_time.stop_id] + params[:arrival_time] = gtfs_time(stop_time.arrival_time) + params[:departure_time] = gtfs_time(stop_time.departure_time) trip_stop_times << GTFSStopTime.new(params) end stop_pattern = trip_stop_times.map(&:stop_id) @@ -394,6 +389,13 @@ def import_trip(trip, stop_times) trip_stop_times.each { |i| i.trip_id = new_trip.id } # Interpolate stop_times GTFSStopTimeService.interpolate_stop_times(trip_stop_times, shape_id) + # Set destinations... + stop_times.each_index do |i| + origin = stop_times[i] + destination = stop_times[i+1] # last stop is nil + origin.destination_id = @stop_ids[destination.stop_id] if destination + origin.destination_arrival_time = gtfs_time(destination.arrival_time) if destination + end # Save stop_times create_chunk(trip_stop_times, 0) end diff --git a/app/services/tile_export_service.rb b/app/services/tile_export_service.rb index dd658bac3..68be63873 100644 --- a/app/services/tile_export_service.rb +++ b/app/services/tile_export_service.rb @@ -73,6 +73,7 @@ def initialize(tilepath, tile, feed_version_ids: nil) # tile unique indexes @route_index = {} @shape_index = {} + @services = {} end def log(msg) @@ -150,6 +151,16 @@ def build_stops return nodes_size end + def get_service(trip) + key = [trip.feed_version_id, trip.service_id] + return @services[key] if @services.key?(key) + calendar = GTFSCalendar.find_by(feed_version_id: trip.feed_version_id, service_id: trip.service_id) || GTFSCalendar.new(feed_version_id: trip.feed_version_id, service_id: trip.service_id) + calendar.start_date ||= calendar.service_added_dates.min + calendar.end_date ||= calendar.service_added_dates.max + @services[key] = calendar + return calendar + end + def build_schedules # Get stop_ids t = Time.now @@ -169,11 +180,15 @@ def build_schedules .where(stop_id: stop_id) .includes(:trip) .find_in_batches do |ssps| + # Evaluate by active feed version - faster than join or where in (?) ssps = ssps.select { |ssp| @feed_version_ids.include?(ssp.feed_version_id) } + # Skip the last stop_time in a trip + ssps = ssps.select(&:destination_id) + # Get unvisited routes for this batch - route_ids = ssps.map { |ssp| ssp.trip.route_id }.select { |route_id| !@route_index.key?(route_id) } + route_ids = ssps.map { |ssp| ssp.trip.route_id }.select { |route_id| !@route_index.key?(route_id) }.uniq ssps.each { |ssp| @route_index[ssp.trip.route_id] ||= nil } # set as visited GTFSRoute.where(id: route_ids).find_each do |route| # get unvisited @route_index[route.id] = base_tile.message.routes.size @@ -182,7 +197,7 @@ def build_schedules end # Get unseen rsps for this batch - rsp_ids = ssps.map { |ssp| ssp.trip.shape_id }.select { |rsp_id| !@shape_index.key?(rsp_id) } + rsp_ids = ssps.map { |ssp| ssp.trip.shape_id }.select { |rsp_id| !@shape_index.key?(rsp_id) }.uniq ssps.each { |ssp| @shape_index[ssp.trip.shape_id] ||= nil } # set as visited GTFSShape.where(id: rsp_ids).find_each do |rsp| shape = make_shape(rsp) @@ -203,6 +218,7 @@ def build_schedules errors[e.class.name.to_sym] += 1 log("error: ssp #{ssp.id}: #{e}") rescue StandardError => e + binding.pry errors[e.class.name.to_sym] += 1 log("error: ssp #{ssp.id}: #{e}") end @@ -260,84 +276,71 @@ def bbox_padded(bbox) # make entity methods def make_stop_pair(ssp) - params = {} - return Valhalla::Mjolnir::Transit::StopPair.new(params) - # TODO: # skip if origin_departure_time < frequency_start_time # skip if bad time information # add < and > to onestop_ids + trip = ssp.trip + calendar = get_service(trip) destination_graphid = @stopid_graphid[ssp.destination_id] origin_graphid = @stopid_graphid[ssp.stop_id] + route_index = @route_index[trip.route_id] + shape_id = @shape_index[trip.shape_id] + + fail InvalidTimeError.new("missing calendar for trip #{trip.trip_id}") unless calendar fail OriginEqualsDestinationError.new("origin_graphid #{origin_graphid} == destination_graphid #{destination_graphid}") if origin_graphid == destination_graphid fail MissingGraphIDError.new("missing origin_graphid for stop #{ssp.stop_id}") unless origin_graphid fail MissingGraphIDError.new("missing destination_graphid for stop #{ssp.destination_id}") unless destination_graphid - - route_index = @route_index[ssp.trip.route_id] - fail MissingRouteError.new("missing route_index for route #{ssp.trip.route_id}") unless route_index - - shape_id = @shape_index[ssp.trip.shape_id] - fail MissingShapeError.new("missing shape for rsp #{ssp.trip.shape_id}") unless shape_id - - trip_id = @trip_index.check(ssp.trip.trip_id) - fail MissingTripError.new("missing trip_id for trip #{ssp.trip.trip_id}") unless trip_id - - destination_arrival_time = seconds_since_midnight(ssp.destination_arrival_time) - origin_departure_time = seconds_since_midnight(ssp.departure_time) - fail InvalidTimeError.new("origin_departure_time #{origin_departure_time} > destination_arrival_time #{destination_arrival_time}") if origin_departure_time > destination_arrival_time - - block_id = @block_index.check(ssp.block_id) + fail MissingRouteError.new("missing route_index for route #{trip.route_id}") unless route_index + fail MissingShapeError.new("missing shape for rsp #{trip.shape_id}") unless shape_id + fail InvalidTimeError.new("origin_departure_time #{ssp.departure_time} > destination_arrival_time #{ssp.destination_arrival_time}") if ssp.departure_time > ssp.destination_arrival_time # Make SSP params = {} # bool bikes_allowed = 1; + (params[:bikes_allowed] = true) if trip.bikes_allowed == 1 # uint32 block_id = 2; - # params[:block_id] = block_id + (params[:block_id] = @block_index.check(trip.block_id)) if trip.block_id # uint32 destination_arrival_time = 3; - params[:destination_arrival_time] = destination_arrival_time + params[:destination_arrival_time] = ssp.destination_arrival_time # uint64 destination_graphid = 4; params[:destination_graphid] = destination_graphid # string destination_onestop_id = 5; - params[:destination_onestop_id] = 's-123-destination' + # params[:destination_onestop_id] = nil # TODO # string operated_by_onestop_id = 6; - params[:operated_by_onestop_id] = 'o-123-test' + # params[:operated_by_onestop_id] = nil # TODO # uint32 origin_departure_time = 7; - params[:origin_departure_time] = origin_departure_time + params[:origin_departure_time] = ssp.departure_time # uint64 origin_graphid = 8; params[:origin_graphid] = origin_graphid # string origin_onestop_id = 9; - params[:origin_onestop_id] = 's-123-origin' + # params[:origin_onestop_id] = nil # TODO # uint32 route_index = 10; params[:route_index] = route_index # repeated uint32 service_added_dates = 11; - params[:service_added_dates] = ssp.service_added_dates.map(&:jd) + params[:service_added_dates] = calendar.service_added_dates.map(&:jd) # repeated bool service_days_of_week = 12; - params[:service_days_of_week] = ssp.service_days_of_week + params[:service_days_of_week] = calendar.service_days_of_week # uint32 service_end_date = 13; - params[:service_end_date] = ssp.service_end_date.jd + params[:service_end_date] = calendar.end_date.jd # repeated uint32 service_except_dates = 14; - params[:service_except_dates] = ssp.service_except_dates.map(&:jd) + params[:service_except_dates] = calendar.service_except_dates.map(&:jd) # uint32 service_start_date = 15; - params[:service_start_date] = ssp.service_start_date.jd + params[:service_start_date] = calendar.start_date.jd # string trip_headsign = 16; - params[:trip_headsign] = ssp.trip_headsign + params[:trip_headsign] = ssp.stop_headsign || trip.trip_headsign # uint32 trip_id = 17; - params[:trip_id] = trip_id + params[:trip_id] = @trip_index.check(trip.id) # bool wheelchair_accessible = 18; - params[:wheelchair_accessible] = true # !!(ssp.wheelchair_accessible) + (params[:wheelchair_accessible] = true) if trip.wheelchair_accessible == 1 # uint32 shape_id = 20; params[:shape_id] = shape_id # float origin_dist_traveled = 21; - params[:origin_dist_traveled] = ssp.origin_dist_traveled if ssp.origin_dist_traveled + (params[:origin_dist_traveled] = ssp.shape_dist_traveled) if ssp.shape_dist_traveled # float destination_dist_traveled = 22; - params[:destination_dist_traveled] = ssp.destination_dist_traveled if ssp.destination_dist_traveled - if ssp.frequency_headway_seconds - # protobuf doesn't define frequency_start_time - # uint32 frequency_end_time = 23; - params[:frequency_end_time] = seconds_since_midnight(ssp.frequency_end_time) - # uint32 frequency_headway_seconds = 24; - params[:frequency_headway_seconds] = ssp.frequency_headway_seconds - end + # params[:destination_dist_traveled] = nil # ssp.destination_dist_traveled if ssp.destination_dist_traveled + # TODO: frequencies + puts params Valhalla::Mjolnir::Transit::StopPair.new(params) end @@ -358,11 +361,11 @@ def make_route(route) # string name = 1; params[:name] = route.name # string onestop_id = 2; - params[:onestop_id] = 'r-123-test' + # params[:onestop_id] = nil # TODO # string operated_by_name = 3; params[:operated_by_name] = route.agency.agency_name # string operated_by_onestop_id = 4; - params[:operated_by_onestop_id] = 'o-123-test' + # params[:operated_by_onestop_id] = nil # TODO # string operated_by_website = 5; params[:operated_by_website] = route.agency.agency_url # uint32 route_color = 6; @@ -374,13 +377,11 @@ def make_route(route) # uint32 route_text_color = 9; params[:route_text_color] = color_to_int(route.route_text_color) # VehicleType vehicle_type = 10; - params[:vehicle_type] = VT::Bus + params[:vehicle_type] = VT::Bus # TODO Valhalla::Mjolnir::Transit::Route.new(params.compact) end def make_node(stop) - onestop_id = 's-123-test' - osm_way_id = 123 params = {} # float lon = 1; params[:lon] = stop.stop_lon @@ -395,25 +396,25 @@ def make_node(stop) # string name = 6; params[:name] = stop.stop_name # string onestop_id = 7; - params[:onestop_id] = onestop_id + # params[:onestop_id] = nil # TODO # uint64 osm_way_id = 8; - params[:osm_way_id] = osm_way_id + # params[:osm_way_id] = nil # TODO # string timezone = 9; - params[:timezone] = stop.stop_timezone + params[:timezone] = stop.stop_timezone || 'America/Los_Angeles' # bool wheelchair_boarding = 10; params[:wheelchair_boarding] = true # bool generated = 11; if stop.location_type == 2 - params[:onestop_id] = "#{onestop_id}>" + # params[:onestop_id] = "#{onestop_id}>" params[:generated] = true end if stop.location_type == 0 - params[:onestop_id] = "#{onestop_id}<" + # params[:onestop_id] = "#{onestop_id}<" # params[:generated] = true # not set for platforms end # uint32 traversability = 12; - if stop.location_type == 2 - params[:traversability] = 3 + if stop.location_type == 2 # TODO: check + params[:traversability] = 3 end Valhalla::Mjolnir::Transit::Node.new(params.compact) end @@ -464,8 +465,8 @@ def self.tile_build_schedules(tilepath, feed_version_ids: nil) def self.export_tiles(tilepath, thread_count: nil, feeds: nil, feed_versions: nil, tiles: nil) # Debug - # ActiveRecord::Base.logger = Logger.new(STDOUT) - # ActiveRecord::Base.logger.level = Logger::DEBUG + ActiveRecord::Base.logger = Logger.new(STDOUT) + ActiveRecord::Base.logger.level = Logger::DEBUG # Filter by feed/feed_version # feed_version_ids = [] @@ -495,7 +496,7 @@ def self.export_tiles(tilepath, thread_count: nil, feeds: nil, feed_versions: ni count_stops << stop.id fvtiles << TileUtils::GraphID.new(level: GRAPH_LEVEL, lon: stop.stop_lon, lat: stop.stop_lat).tile else - stop_platforms[stop.parent_stop_id] << stop.id + stop_platforms[stop.parent_station_id] << stop.id end end puts "\t(#{count}/#{total}) #{feed_version.feed.onestop_id} #{feed_version.sha1}: #{fvtiles.size} tiles" diff --git a/app/services/tile_export_service_old.rb b/app/services/tile_export_service_old.rb index 2a548a184..e62e739ba 100644 --- a/app/services/tile_export_service_old.rb +++ b/app/services/tile_export_service_old.rb @@ -1,4 +1,4 @@ -module TileExportService +module TileExportServiceOld BBOX_PADDING = 0.1 KEY_QUEUE_STOPS = 'queue_stops' KEY_QUEUE_SCHEDULES = 'queue_schedules' From 404bf0a9dd6f3afcc2b3d2afde194d940458e5ab Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Wed, 8 Aug 2018 17:56:06 -0700 Subject: [PATCH 04/14] Add useful methods to GTFSCalendar --- app/models/gtfs_calendar.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/app/models/gtfs_calendar.rb b/app/models/gtfs_calendar.rb index 3588004bb..c2fee5949 100644 --- a/app/models/gtfs_calendar.rb +++ b/app/models/gtfs_calendar.rb @@ -40,4 +40,16 @@ class GTFSCalendar < ActiveRecord::Base validates :service_id, presence: true validates :monday, :tuesday, :wednesday, :thursday, :friday, :saturday, :sunday, inclusion: { in: [true, false] } has_many :exceptions, -> (c) { where("gtfs_calendar_dates.feed_version_id = :feed_version_id", feed_version_id: c.feed_version_id) }, class_name: 'GTFSCalendarDate', primary_key: 'service_id', foreign_key: :service_id + + def service_added_dates + exceptions.map { |i| i.date if i.exception_type == 1 }.compact.uniq + end + + def service_except_dates + exceptions.map { |i| i.date if i.exception_type == 2 }.compact.uniq + end + + def service_days_of_week + [monday, tuesday, wednesday, thursday, friday, saturday, sunday] + end end From 265bcfe504d60297a6a420fd1b52ebe270aa061a Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Tue, 14 Aug 2018 11:51:22 -0700 Subject: [PATCH 05/14] calendar/calendar_dates cache --- app/models/gtfs_trip.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/models/gtfs_trip.rb b/app/models/gtfs_trip.rb index 0b5a495fa..b08532b54 100644 --- a/app/models/gtfs_trip.rb +++ b/app/models/gtfs_trip.rb @@ -35,7 +35,10 @@ class GTFSTrip < ActiveRecord::Base include GTFSEntity has_many :stop_times, class_name: 'GTFSStopTime', foreign_key: 'trip_id' has_many :stops, -> { distinct }, through: :stop_times - + attr_accessor :calendar, :calendar_dates + # has_one :calendar, -> (trip) { where( feed_version_id: trip.feed_version_id ) }, class_name: 'GTFSCalendar', primary_key: 'service_id', foreign_key: 'service_id' + # has_many :calendar_dates, -> (trip) { where( feed_version_id: trip.feed_version_id ) }, class_name: 'GTFSCalendarDate', primary_key: 'service_id', foreign_key: 'service_id' + def service c = GTFSCalendar.find_by(feed_version_id: feed_version_id, service_id: service_id) || GTFSCalendar.new(feed_version_id: feed_version_id, service_id: service_id) { @@ -47,7 +50,6 @@ def service } end - belongs_to :route, class_name: 'GTFSRoute' belongs_to :feed_version belongs_to :entity, class_name: 'RouteStopPattern' From b812db56dc44e41d967a08dd7c42f74fe3e32c6f Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Tue, 14 Aug 2018 11:51:36 -0700 Subject: [PATCH 06/14] performance improvements --- app/services/tile_export_service.rb | 91 +++++++++++++++++------------ 1 file changed, 53 insertions(+), 38 deletions(-) diff --git a/app/services/tile_export_service.rb b/app/services/tile_export_service.rb index 68be63873..08cf9f14e 100644 --- a/app/services/tile_export_service.rb +++ b/app/services/tile_export_service.rb @@ -73,7 +73,6 @@ def initialize(tilepath, tile, feed_version_ids: nil) # tile unique indexes @route_index = {} @shape_index = {} - @services = {} end def log(msg) @@ -97,6 +96,7 @@ def build_stops .where(feed_version_id: @feed_version_ids) .where(parent_station_id: nil) .geometry_within_bbox(bbox_padded(tile.bbox)) + .includes(:children) .find_each do |stop| # Check if stop is inside tile @@ -151,22 +151,14 @@ def build_stops return nodes_size end - def get_service(trip) - key = [trip.feed_version_id, trip.service_id] - return @services[key] if @services.key?(key) - calendar = GTFSCalendar.find_by(feed_version_id: trip.feed_version_id, service_id: trip.service_id) || GTFSCalendar.new(feed_version_id: trip.feed_version_id, service_id: trip.service_id) - calendar.start_date ||= calendar.service_added_dates.min - calendar.end_date ||= calendar.service_added_dates.max - @services[key] = calendar - return calendar - end - def build_schedules # Get stop_ids t = Time.now tileset = TileUtils::TileSet.new(@tilepath) base_tile = tileset.read_tile(GRAPH_LEVEL, @tile) stop_ids = base_tile.message.nodes.map { |node| @graphid_stopid[node.graphid] }.compact + trips = {} + calendars = {} # Build stop_pairs for each stop_id tile_ext = 0 @@ -178,50 +170,73 @@ def build_schedules stop_pairs_stop_id_count = 0 GTFSStopTime .where(stop_id: stop_id) - .includes(:trip) .find_in_batches do |ssps| - # Evaluate by active feed version - faster than join or where in (?) - ssps = ssps.select { |ssp| @feed_version_ids.include?(ssp.feed_version_id) } - # Skip the last stop_time in a trip ssps = ssps.select(&:destination_id) - # Get unvisited routes for this batch + # Cache: Get unseen trips + trip_ids = ssps.map(&:trip_id).select { |t| !trips.key?(t) } + GTFSTrip.find(trip_ids).each do |t| + trips[t.id] = t + end + ssps.each { |ssp| ssp.trip = trips[ssp.trip_id] } + + # Cache: Get unseen calendars and calendar_dates + ssps.each do |ssp| + # if we don't have the calendar cached, find the calendar or create an empty one + args = {feed_version_id: ssp.trip.feed_version_id, service_id: ssp.trip.service_id} + key = [ssp.trip.feed_version_id, ssp.trip.service_id] + calendar = calendars[key] + if calendar.nil? + calendar = GTFSCalendar.find_by(args) # || GTFSCalendar.new(args) + end + calendar.start_date ||= calendar.service_added_dates.min + calendar.end_date ||= calendar.service_added_dates.max + ssp.trip.calendar = calendar + calendars[key] = calendar + end + + # Get unvisited routes for this batch, add to tile route_ids = ssps.map { |ssp| ssp.trip.route_id }.select { |route_id| !@route_index.key?(route_id) }.uniq ssps.each { |ssp| @route_index[ssp.trip.route_id] ||= nil } # set as visited - GTFSRoute.where(id: route_ids).find_each do |route| # get unvisited - @route_index[route.id] = base_tile.message.routes.size - debug("route: #{route.id} -> #{@route_index[route.id]}") - base_tile.message.routes << make_route(route) + if route_ids.size > 0 + GTFSRoute.where(id: route_ids).find_each do |route| # get unvisited + @route_index[route.id] = base_tile.message.routes.size + debug("route: #{route.id} -> #{@route_index[route.id]}") + base_tile.message.routes << make_route(route) + end end - # Get unseen rsps for this batch - rsp_ids = ssps.map { |ssp| ssp.trip.shape_id }.select { |rsp_id| !@shape_index.key?(rsp_id) }.uniq + # Get unseen shapes for this batch, add to tile + shape_ids = ssps.map { |ssp| ssp.trip.shape_id }.select { |shape_id| !@shape_index.key?(shape_id) }.uniq ssps.each { |ssp| @shape_index[ssp.trip.shape_id] ||= nil } # set as visited - GTFSShape.where(id: rsp_ids).find_each do |rsp| - shape = make_shape(rsp) - shape.shape_id = base_tile.message.shapes.size + 1 - @shape_index[rsp.id] = shape.shape_id - debug("shape: #{rsp.id} -> #{@shape_index[rsp.id]}") - base_tile.message.shapes << shape + if shape_ids.size > 0 + GTFSShape.where(id: shape_ids).find_each do |shape| + pshape = make_shape(shape) + pshape.shape_id = base_tile.message.shapes.size + 1 + @shape_index[shape.id] = pshape.shape_id + debug("shape: #{shape.id} -> #{@shape_index[shape.id]}") + base_tile.message.shapes << pshape + end end # Process each ssp ssps.each do |ssp| # process ssp and count errors begin - stop_pairs_tile.message.stop_pairs << make_stop_pair(ssp) - stop_pairs_stop_id_count += 1 - stop_pairs_total += 1 + i = make_stop_pair(ssp) rescue TileValueError => e errors[e.class.name.to_sym] += 1 log("error: ssp #{ssp.id}: #{e}") rescue StandardError => e - binding.pry errors[e.class.name.to_sym] += 1 log("error: ssp #{ssp.id}: #{e}") end + next unless i + stop_pairs_tile.message.stop_pairs << i + stop_pairs_stop_id_count += 1 + stop_pairs_total += 1 end # Write supplement tile, start new tile @@ -281,7 +296,7 @@ def make_stop_pair(ssp) # skip if bad time information # add < and > to onestop_ids trip = ssp.trip - calendar = get_service(trip) + calendar = ssp.trip.calendar destination_graphid = @stopid_graphid[ssp.destination_id] origin_graphid = @stopid_graphid[ssp.stop_id] route_index = @route_index[trip.route_id] @@ -292,7 +307,7 @@ def make_stop_pair(ssp) fail MissingGraphIDError.new("missing origin_graphid for stop #{ssp.stop_id}") unless origin_graphid fail MissingGraphIDError.new("missing destination_graphid for stop #{ssp.destination_id}") unless destination_graphid fail MissingRouteError.new("missing route_index for route #{trip.route_id}") unless route_index - fail MissingShapeError.new("missing shape for rsp #{trip.shape_id}") unless shape_id + fail MissingShapeError.new("missing shape for shape #{trip.shape_id}") unless shape_id fail InvalidTimeError.new("origin_departure_time #{ssp.departure_time} > destination_arrival_time #{ssp.destination_arrival_time}") if ssp.departure_time > ssp.destination_arrival_time # Make SSP @@ -340,16 +355,16 @@ def make_stop_pair(ssp) # float destination_dist_traveled = 22; # params[:destination_dist_traveled] = nil # ssp.destination_dist_traveled if ssp.destination_dist_traveled # TODO: frequencies - puts params + # puts params Valhalla::Mjolnir::Transit::StopPair.new(params) end - def make_shape(rsp) + def make_shape(shape) params = {} # uint32 shape_id = 1; # bytes encoded_shape = 2; # reverse coordinates - reversed = rsp.geometry[:coordinates].map { |a,b| [b,a] } + reversed = shape.geometry[:coordinates].map { |a,b| [b,a] } params[:encoded_shape] = TileUtils::Shape7.encode(reversed) Valhalla::Mjolnir::Transit::Shape.new(params) end @@ -477,7 +492,7 @@ def self.export_tiles(tilepath, thread_count: nil, feeds: nil, feed_versions: ni # else # feed_version_ids = Feed.where_active_feed_version_import_level(IMPORT_LEVEL).pluck(:active_feed_version_id) # end - feed_version_ids = FeedVersion.pluck(:id) + feed_version_ids = [3] # FeedVersion.pluck(:id) # Build bboxes puts "Selecting tiles..." From fbcbf681b9b1eaa1c97eb0978c58d5ef815e02dd Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Tue, 14 Aug 2018 11:51:56 -0700 Subject: [PATCH 07/14] performance improvements --- app/services/gtfs_import_service.rb | 92 +++++++++++++++-------------- 1 file changed, 48 insertions(+), 44 deletions(-) diff --git a/app/services/gtfs_import_service.rb b/app/services/gtfs_import_service.rb index 830150f9a..554bf1fc9 100644 --- a/app/services/gtfs_import_service.rb +++ b/app/services/gtfs_import_service.rb @@ -40,16 +40,16 @@ def import t_parse = Time.now log('finding selected entities...') # list of agency_ids to import - log('...agencies') + log(' agencies') selected_agency_ids = Set.new default_agency_id = nil @gtfs.each_agency do |e| default_agency_id ||= e.agency_id selected_agency_ids << e.agency_id end - log("...default_agency_id: #{default_agency_id}") + log(" default_agency_id: #{default_agency_id}") # agency associated routes - log('...routes') + log(' routes') selected_route_ids = Set.new @gtfs.each_route do |e| e.agency_id ||= default_agency_id @@ -57,14 +57,14 @@ def import selected_route_ids << e.id end # trips associated with selected routes - log('...trips') + log(' trips') selected_trip_ids = Set.new @gtfs.each_trip do |e| next unless selected_route_ids.include?(e.route_id) selected_trip_ids << e.id end # stops associated with selected trips, and trip counter for pruning - log('...stops') + log(' stops') selected_stop_ids = Set.new trip_stop_counter = Hash.new { |h,k| h[k] = 0 } stop_time_counter = 0 @@ -80,7 +80,7 @@ def import selected_stop_ids << e.parent_station if e.parent_station end # pass through trips again for services and shapes - log('...services, pruning trips') + log(' services, pruning trips') selected_service_ids = Set.new selected_shape_ids = Set.new @gtfs.each_trip do |e| @@ -92,7 +92,7 @@ def import end end # shapes - log('...shapes') + log(' shapes') shape_counter = Hash.new { |h,k| h[k] = 0 } if @gtfs.file_present?('shapes.txt') @gtfs.each_shape do |e| @@ -101,7 +101,7 @@ def import end end # Fares and transfers - log("...time: #{((Time.now-t_parse).round(2))}") + log(" time: #{((Time.now-t_parse).round(2))}") # Import time('agencies') { import_agencies(selected_agency_ids) } time('stops') { import_stops(selected_stop_ids) } @@ -304,44 +304,48 @@ def import_fare_attributes(default_agency_id=nil) def import_shapes(shape_counter=nil) return unless @gtfs.file_present?('shapes.txt') # load shapes in chunks - f = GTFSShape.geofactory yield_chunks(shape_counter, SHAPE_CHUNK_SIZE) do |shape_id_chunk| - log("processing shape_id_chunks: #{shape_id_chunk.size}") + log("processing shape_id_chunks: #{shape_id_chunk.size} shape_lines") @gtfs.each_shape_line(shape_id_chunk) do |shape_line| - log("processing shape_line: #{shape_line.shape_id} shapes #{shape_line.shapes.size}") - params = {} - params[:feed_version_id] = @feed_version.id - params[:shape_id] = shape_line.shape_id - params[:geometry] = f.line_string( - shape_line.shapes.map { |s| - f.point( - gtfs_float(s.shape_pt_lon), - gtfs_float(s.shape_pt_lat), - gtfs_float(s.shape_dist_traveled) - ) - } - ) - create(GTFSShape.new(params), shape_line.shape_id, @shape_ids) + time(" shape_line #{shape_line.shape_id} shapes #{shape_line.shapes.size}") { import_shape_line(shape_line) } end end end + def import_shape_line(shape_line) + f = GTFSShape.geofactory + params = {} + params[:feed_version_id] = @feed_version.id + params[:shape_id] = shape_line.shape_id + params[:geometry] = f.line_string( + shape_line.shapes.map { |s| + f.point( + gtfs_float(s.shape_pt_lon), + gtfs_float(s.shape_pt_lat), + gtfs_float(s.shape_dist_traveled) + ) + } + ) + create(GTFSShape.new(params), shape_line.shape_id, @shape_ids) +end + def import_trips_and_stop_times(trip_stop_counter=nil) - # load trips + # Load trips @gtfs.trips - # stop_pattern shape_ids + # Stop_pattern shape_ids @stop_pattern_shape_ids = {} - # load stop_times in chunks + # Cache distances by shape_id + @distances = {} + # Load stop_times in chunks yield_chunks(trip_stop_counter, STOP_TIME_CHUNK_SIZE) do |trip_id_chunk| - log("processing trip_id_chunks: #{trip_id_chunk.size}") + log("processing trip_id_chunks: #{trip_id_chunk.size} trips") @gtfs.each_trip_stop_times(trip_id_chunk) do |trip_id, stop_times| - import_trip(@gtfs.trip(trip_id), stop_times) + time(" trip #{trip_id} stop_times #{stop_times.size}") { import_trip(@gtfs.trip(trip_id), stop_times) } end end end def import_trip(trip, stop_times) - log("processing trip: #{trip.id} stop_times #{stop_times.size}") # Create stop_times trip_stop_times = [] stop_times.each do |stop_time| @@ -354,14 +358,15 @@ def import_trip(trip, stop_times) params[:drop_off_type] = gtfs_int(stop_time.drop_off_type) || 0 params[:shape_dist_traveled] = gtfs_float(stop_time.shape_dist_traveled) params[:timepoint] = gtfs_int(stop_time.timepoint) - # where params[:stop_id] = @stop_ids[stop_time.stop_id] params[:arrival_time] = gtfs_time(stop_time.arrival_time) params[:departure_time] = gtfs_time(stop_time.departure_time) trip_stop_times << GTFSStopTime.new(params) end + trip_stop_times = trip_stop_times.sort_by(&:stop_sequence) stop_pattern = trip_stop_times.map(&:stop_id) # Create trip + return unless trip_stop_times.size > 1 return unless @route_ids[trip.route_id] params = {} params[:feed_version_id] = @feed_version.id @@ -384,17 +389,17 @@ def import_trip(trip, stop_times) # Save trip new_trip = create(GTFSTrip.new(params), trip.trip_id, @trip_ids) return unless new_trip - return unless trip_stop_times.size > 0 + # Interpolate stop_times + # TODO: interpolate & validate before saving trip?? rescue exception? + d = @distances[shape_id] || {} + @distances[shape_id] = d + GTFSStopTimeService.interpolate_stop_times(trip_stop_times, shape_id, distances=d) # Assign trip_id to stop_times trip_stop_times.each { |i| i.trip_id = new_trip.id } - # Interpolate stop_times - GTFSStopTimeService.interpolate_stop_times(trip_stop_times, shape_id) - # Set destinations... - stop_times.each_index do |i| - origin = stop_times[i] - destination = stop_times[i+1] # last stop is nil - origin.destination_id = @stop_ids[destination.stop_id] if destination - origin.destination_arrival_time = gtfs_time(destination.arrival_time) if destination + # Set destinations + trip_stop_times[0..-2].zip(trip_stop_times[1..-1]).each do |o,d| + o.destination_id = d.stop_id + o.destination_arrival_time = d.arrival_time end # Save stop_times create_chunk(trip_stop_times, 0) @@ -423,8 +428,7 @@ def create_chunk(chunk, chunk_size=nil, filter=false) if chunk.size > chunk_size chunk.each { |i| i.skip_association_validations = true } chunk, invalid = chunk.partition(&:valid?) - invalid.each { |i| log(" invalid: #{i.class.name} #{i.to_json}"); puts i.errors.messages } - # log(" import #{chunk.size}") + invalid.each { |i| log(" invalid: #{i.class.name} #{i.to_json}"); puts i.errors.messages } chunk.first.class.import(chunk) if chunk.size > 0 chunk = [] end @@ -438,7 +442,7 @@ def create(record, idid=nil, idmap=nil) record.save! # log(" saved: #{record.class.name} #{record.to_json}") rescue ActiveRecord::RecordInvalid, ActiveRecord::StatementInvalid => e - log(" failed: #{record.class.name} #{record.to_json}: #{record.errors}") + log(" failed: #{record.class.name} #{record.to_json}: #{record.errors}") return nil end idmap[idid] = record.id if idmap @@ -457,7 +461,7 @@ def import_log def time(msg, &block) t = Time.now block.call - log("#{msg}: #{((Time.now-t)).round(2)}s") + log("#{msg}: #{((Time.now-t)).round(4)}s") end def gtfs_int(value) From 5b95dd0118101a3b21ba7daa910f901920d9318f Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Tue, 14 Aug 2018 11:52:09 -0700 Subject: [PATCH 08/14] WIP --- app/services/gtfs_stop_time_service.rb | 264 +++++++++++++++---------- 1 file changed, 161 insertions(+), 103 deletions(-) diff --git a/app/services/gtfs_stop_time_service.rb b/app/services/gtfs_stop_time_service.rb index 47c0f3215..f439361f1 100644 --- a/app/services/gtfs_stop_time_service.rb +++ b/app/services/gtfs_stop_time_service.rb @@ -1,84 +1,180 @@ class GTFSStopTimeService - def self.debug(msg) - log(msg) - end def self.clean_stop_times(stop_times) # Sort by stop_sequence stop_times.sort_by! { |st| st.stop_sequence } - # If we only have 1 time, assume it is both arrival and departure + times = [] + distances = [] stop_times.each do |st| (st.arrival_time = st.departure_time) if st.arrival_time.nil? (st.departure_time = st.arrival_time) if st.departure_time.nil? + times << st.arrival_time if st.arrival_time + times << st.departure_time if st.departure_time + distances << st.shape_dist_traveled if st.shape_dist_traveled end - - # Ensure time is positive - current = stop_times.first.arrival_time - stop_times.each do |st| - s = st.arrival_time - fail Exception.new('cannot go backwards in time') if s && s < current - current = s if s - s = st.departure_time - fail Exception.new('cannot go backwards in time') if s && s < current - current = s if s - end - # These two values are required by spec - fail Exception.new('missing first departure time') if stop_times.first.departure_time.nil? - fail Exception.new('missing last arrival time') if stop_times.last.arrival_time.nil? + return [] if stop_times.first.departure_time.nil? + return [] if stop_times.last.arrival_time.nil? + # Ensure shape_dist_traveled is increasing + return [] unless distances == distances.sort + # Ensure time is increasing + return [] unless times == times.sort + # OK return stop_times end - def self.interpolate_stop_times(stop_times, shape_id) + def self.interpolate_stop_times(stop_times, shape_id, d1=nil, d2=nil) + # Tidy up our stop_times stop_times = clean_stop_times(stop_times) - # Return early if possible - gaps = interpolate_find_gaps(stop_times) - return stop_times if gaps.size == 0 - # Measure stops along line - trip_pattern = stop_times.map(&:stop_id) - # First pass: line interpolation - distances = get_shape_stop_distances(trip_pattern, shape_id) - gaps.each do |gap| - o, c = gap - interpolate_gap_distance(stop_times[o..c], distances) + + # Measure shape length and stop distances (and cache) + d1 ||= {} + measure_stops = stop_times.map(&:stop_id).select { |i| d1[i].nil? } + d1 = get_shape_stop_distances(measure_stops, shape_id, distances=d1) + + # Do we have values for shape_dist_traveled on stop_times AND shape? + if stop_times.map(&:shape_dist_traveled).all? + s1 = stop_times.map(&:shape_dist_traveled) + s2 = GTFSShape.find(shape_id).geometry[:coordinates].map { |c| c[2] } + s2d = s2.last - s2.first + shape_length = d1[nil] + if s2.all? && s2d > 0 && shape_length && shape_length > 0 + # Convert stop_times shape_dist_traveled to meters + cm = shape_length / s2d + stop_times.each { |st| stop_times.shape_dist_traveled *= cm } + else + # Reset shape_dist_traveled + stop_times.each { |st| stop_times.shape_dist_traveled = nil } + end + else + # Reset shape_dist_traveled + stop_times.each { |st| stop_times.shape_dist_traveled = nil } end - # Second pass: distance interpolation - gaps = interpolate_find_gaps(stop_times) - gaps.each do |gap| - o, c = gap - interpolate_gap_linear(stop_times[o..c]) + + # Do we need to fall back to linear stop-stop distances? + distances = stop_times.map { |st| d1[st.stop_id] } + distances.reverse! if distances.first > distances.last + if distances != distances.sort + d1 = get_linear_stop_distances(trip_pattern) end + + stop_times.each { |st| st.shape_dist_traveled = nil } + stop_times.first.shape_dist_traveled = distances[stop_times.first.stop_id] + stop_times.last.shape_dist_traveled = distances[stop_times.last.stop_id] + + # Fill in dist gaps. First pass: distance; second pass: linear + interpolate_find_dist(stop_times).each { |o,c| interpolate_distance(stop_times[o..c], d1) } + # Fill in times + interpolate_find_time(stop_times).each { |o,c| interpolate_time(stop_times[o..c]) } return stop_times end - def self.get_shape_stop_distances(trip_pattern, shape_id) + def self.get_linear_stop_distances(trip_pattern, distances=nil) + distances ||= {} + return distances unless trip_pattern.size > 0 + trip_pattern = trip_pattern.map(&:to_i) + # Raw SQL, but difficult to get cte's otherwise. + s = <<-EOF + WITH + gtfs_stops AS ( + SELECT id, geometry::geometry + FROM gtfs_stops + INNER JOIN ( + SELECT unnest, ordinality + FROM unnest( ARRAY[#{trip_pattern.join(',')}] ) WITH ORDINALITY + ) as unnest + ON gtfs_stops.id = unnest + ORDER BY ordinality + ), + shapes AS ( + SELECT ST_Length(ST_MakeLine(geometry)::geography) AS shape_length, ST_MakeLine(geometry) AS geometry FROM gtfs_stops + ) + SELECT + gtfs_stops.id, + shapes.shape_length, + ST_LineLocatePoint(shapes.geometry::geometry, gtfs_stops.geometry::geometry) AS shape_percent + FROM gtfs_stops + INNER JOIN shapes ON true + EOF + GTFSStop.find_by_sql(s.squish).each do |row| + distances[nil] ||= row.shape_length + distances[row.id] = row.shape_percent + end + return distances + end + + def self.get_shape_stop_distances(trip_pattern, shape_id, distances=nil) # Calculate line percent from closest point to stop - distances = {} - s = 'gtfs_stops.id, ST_LineLocatePoint(shapes.geometry::geometry, ST_ClosestPoint(shapes.geometry::geometry, gtfs_stops.geometry::geometry)) AS line_s' - g = GTFSStop.select(s) - # Create shape if necessary - # if shape_id - g = g.joins('INNER JOIN gtfs_shapes AS shapes ON true') - # else - # shape_id = 0 - # g = g.joins("INNER JOIN (SELECT 0 as id, ST_MakeLine(geometry) AS geometry FROM (SELECT geometry FROM gtfs_stops INNER JOIN (SELECT unnest,ordinality FROM unnest( ARRAY[#{trip_pattern.join(',')}] ) WITH ORDINALITY) as unnest ON gtfs_stops.id = unnest ORDER BY ordinality) as q) AS shapes ON true") - # end - # Filter - g = g.where('shapes.id': shape_id, id: trip_pattern) - # Run - g.each do |row| - distances[row.id] = row.line_s + distances ||= {} + return distances unless trip_pattern.size > 0 + shape_id = shape_id.to_i + trip_pattern = trip_pattern.map(&:to_i) + # Raw SQL, as above. + s = <<-EOF + WITH + shapes AS ( + SELECT + gtfs_shapes.id, + gtfs_shapes.geometry, + ST_Length(gtfs_shapes.geometry) as shape_length + FROM gtfs_shapes + WHERE gtfs_shapes.id = #{shape_id} + ) + SELECT + gtfs_stops.id, + shapes.shape_length, + ST_LineLocatePoint(shapes.geometry::geometry, gtfs_stops.geometry::geometry) AS shape_percent + FROM gtfs_stops + INNER JOIN shapes ON true + WHERE gtfs_stops.id IN (#{trip_pattern.join(',')}); + EOF + GTFSStop.find_by_sql(s.squish).each do |row| + distances[nil] ||= row.shape_length + distances[row.id] = row.shape_percent end return distances end - def self.interpolate_find_gaps(stop_times) + def self.interpolate_time(stop_times) + o_distance = stop_times.first.shape_dist_traveled + c_distance = stop_times.last.shape_dist_traveled + o_time = stop_times.first.departure_time + c_time = stop_times.last.arrival_time + stop_times.each do |st| + next if st.arrival_time && st.departure_time + pct = (st.shape_dist_traveled - o_distance) / (c_distance - o_distance) + st.arrival_time = st.departure_time = (pct * (c_time - o_time)) + o_time + st.interpolated += 10 + end + end + + def self.interpolate_distance(stop_times, distances) + # check that we can interpolate reasonably + d = stop_times.map { |st| st.shape_dist_travelled || distances[st.stop_id] } + return unless d == d.sort && d.all? + stop_times.zip(d).each do |st,i| + next if st.shape_dist_traveled + st.shape_dist_traveled = i + st.interpolated = 1 + end + end + + def self.interpolate_linear(stop_times) + o, c = stop_times.first.shape_dist_traveled, stop_times.last.shape_dist_traveled + increment = (c - o) / (stop_times.size.to_f-1) + stop_times.each_with_index do |st,i| + next if st.shape_dist_traveled + st.shape_dist_traveled = increment * i + o + st.interpolated = 2 + end + end + + def self.interpolate_find_time(stop_times) gaps = [] o, c = nil, nil stop_times.each_with_index do |st, i| # close an open gap - # puts "i: #{i} st: #{st.stop_sequence} stop: #{st.stop_id} arrival_time: #{st.arrival_time} departure_time: #{st.departure_time}" if o && st.arrival_time gaps << [o, i] if (i-o > 1) o = nil @@ -91,56 +187,18 @@ def self.interpolate_find_gaps(stop_times) return gaps end - def self.interpolate_gap_distance(stop_times, distances) - # debug("trip: #{stop_times.first.trip_id} interpolate_gap_distance: #{stop_times.first.stop_sequence} -> #{stop_times.last.stop_sequence}") - # open and close times - o_time = stop_times.first.departure_time - c_time = stop_times.last.arrival_time - # open and close distances - o_distance = distances[stop_times.first.stop_id] - c_distance = distances[stop_times.last.stop_id] - # check that we can interpolate reasonably - p_distance = o_distance - stop_times.each do |st| - i_distance = distances[st.stop_id] - return unless i_distance - return if i_distance < p_distance # cannot backtrack - return if i_distance > c_distance # cannot exceed end - p_distance = i_distance - end - # interpolate on distance - # debug("\tlength: #{c_distance - o_distance} duration: #{c_time - o_time}") - # debug("\to_distance: #{o_distance} o_time: #{o_time}") - stop_times[1...-1].each do |st| - i_distance = distances[st.stop_id] - pct = (i_distance - o_distance) / (c_distance - o_distance) - i_time = (c_time - o_time) * pct + o_time - # debug("\ti_distance: #{i_distance} pct: #{pct} i_time: #{i_time}") - st.arrival_time = i_time - st.departure_time = i_time - st.interpolated = 1 - end - # debug("\tc_distance: #{c_distance} c_time: #{c_time}") - return true - end - - def self.interpolate_gap_linear(stop_times) - # debug("trip: #{stop_times.first.trip_id} interpolate_gap_linear: #{stop_times.first.stop_sequence} -> #{stop_times.last.stop_sequence}") - # open and close times - o_time = stop_times.first.departure_time - c_time = stop_times.last.arrival_time - # interpolate on time - p_time = o_time - # debug("\tduration: #{c_time - o_time}") - # debug("\ti: 0 o_time: #{o_time}") - stop_times[1...-1].each_with_index do |st,i| - pct = pct = (i+1) / (stop_times.size.to_f-1) - i_time = (c_time - o_time) * pct + o_time - # debug("\ti: #{i+1} pct: #{pct} i_time: #{i_time} ") - st.arrival_time = i_time - st.departure_time = i_time - st.interpolated = 2 + def self.interpolate_find_dist(stop_times) + gaps = [] + o, c = nil, nil + stop_times.each_with_index do |st, i| + if o && st.shape_dist_traveled + gaps << [o, i] if (i-o > 1) + o = nil + end + if o.nil? && st.shape_dist_traveled + o = i + end end - # debug("\ti: #{stop_times.size-1} c_time: #{c_time}") - end + return gaps + end end From 41839f73b81c53fe83f8a4cfdfb2578ee8961f7e Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Tue, 14 Aug 2018 23:59:04 -0700 Subject: [PATCH 09/14] WIP --- app/services/gtfs_stop_time_service.rb | 221 +++++++++++++++++-------- 1 file changed, 155 insertions(+), 66 deletions(-) diff --git a/app/services/gtfs_stop_time_service.rb b/app/services/gtfs_stop_time_service.rb index f439361f1..8cf6d0215 100644 --- a/app/services/gtfs_stop_time_service.rb +++ b/app/services/gtfs_stop_time_service.rb @@ -70,72 +70,6 @@ def self.interpolate_stop_times(stop_times, shape_id, d1=nil, d2=nil) return stop_times end - def self.get_linear_stop_distances(trip_pattern, distances=nil) - distances ||= {} - return distances unless trip_pattern.size > 0 - trip_pattern = trip_pattern.map(&:to_i) - # Raw SQL, but difficult to get cte's otherwise. - s = <<-EOF - WITH - gtfs_stops AS ( - SELECT id, geometry::geometry - FROM gtfs_stops - INNER JOIN ( - SELECT unnest, ordinality - FROM unnest( ARRAY[#{trip_pattern.join(',')}] ) WITH ORDINALITY - ) as unnest - ON gtfs_stops.id = unnest - ORDER BY ordinality - ), - shapes AS ( - SELECT ST_Length(ST_MakeLine(geometry)::geography) AS shape_length, ST_MakeLine(geometry) AS geometry FROM gtfs_stops - ) - SELECT - gtfs_stops.id, - shapes.shape_length, - ST_LineLocatePoint(shapes.geometry::geometry, gtfs_stops.geometry::geometry) AS shape_percent - FROM gtfs_stops - INNER JOIN shapes ON true - EOF - GTFSStop.find_by_sql(s.squish).each do |row| - distances[nil] ||= row.shape_length - distances[row.id] = row.shape_percent - end - return distances - end - - def self.get_shape_stop_distances(trip_pattern, shape_id, distances=nil) - # Calculate line percent from closest point to stop - distances ||= {} - return distances unless trip_pattern.size > 0 - shape_id = shape_id.to_i - trip_pattern = trip_pattern.map(&:to_i) - # Raw SQL, as above. - s = <<-EOF - WITH - shapes AS ( - SELECT - gtfs_shapes.id, - gtfs_shapes.geometry, - ST_Length(gtfs_shapes.geometry) as shape_length - FROM gtfs_shapes - WHERE gtfs_shapes.id = #{shape_id} - ) - SELECT - gtfs_stops.id, - shapes.shape_length, - ST_LineLocatePoint(shapes.geometry::geometry, gtfs_stops.geometry::geometry) AS shape_percent - FROM gtfs_stops - INNER JOIN shapes ON true - WHERE gtfs_stops.id IN (#{trip_pattern.join(',')}); - EOF - GTFSStop.find_by_sql(s.squish).each do |row| - distances[nil] ||= row.shape_length - distances[row.id] = row.shape_percent - end - return distances - end - def self.interpolate_time(stop_times) o_distance = stop_times.first.shape_dist_traveled c_distance = stop_times.last.shape_dist_traveled @@ -201,4 +135,159 @@ def self.interpolate_find_dist(stop_times) end return gaps end + + def self.get_shape_stop_segments(trip_pattern, shape_id, cache=nil, maxdistance=200, segments=100) + # Calculate line percent from closest point to stop + cache ||= {} + trip_pattern = trip_pattern.select { |i| !cache.key?(i) } + return cache unless trip_pattern.size > 0 + shape_id = shape_id.to_i + trip_pattern = trip_pattern.map(&:to_i) + maxdistance = maxdistance.to_i + segments = segments.to_i + # a: get the shape, as geometry + # linesubs: break shape into 100 segments, as geography + # segs: get cumulative line distance, including the current segment + # gtfs_stops: get stops + # gtfs_stops_segs: find distance between each stop and each segment + s = <<-EOF + WITH + a AS ( + SELECT + gtfs_shapes.id, + gtfs_shapes.geometry::geometry as geometry, + ST_Length(gtfs_shapes.geometry) as shape_length + FROM gtfs_shapes WHERE id = #{shape_id} + ), + linesubs AS ( + SELECT + i AS seg_id, + ST_LineSubstring(a.geometry, (i::float)/#{segments},(i+1::float)/#{segments})::geography AS shape, + shape_length + FROM a, GENERATE_SERIES(0,#{segments-1}) AS i + ), + segs AS ( + SELECT + seg_id, + shape, + shape_length, + ST_Length(shape) as seg_length, + sum(ST_Length(shape)) OVER (ORDER BY seg_id) AS seg_length_sum + FROM linesubs + ), + gtfs_stops AS ( + SELECT + gtfs_stops.id, + gtfs_stops.geometry + FROM gtfs_stops + WHERE gtfs_stops.id IN (#{trip_pattern.join(',')}) + ), + gtfs_stops_segs AS ( + SELECT + gtfs_stops.id, + seg_id, + segs.seg_length, + segs.shape_length as shape_length, + (segs.seg_length_sum - segs.seg_length) AS seg_length_sum, + ST_LineLocatePoint(segs.shape::geometry, gtfs_stops.geometry::geometry) AS seg_percent, + ST_Distance(segs.shape, gtfs_stops.geometry) AS seg_distance + FROM gtfs_stops + INNER JOIN segs ON true + ) + SELECT + * + FROM gtfs_stops_segs + WHERE seg_distance < #{maxdistance} + EOF + # s = s.squish + trip_pattern.each { |i| cache[i] ||= [] } + GTFSStop.find_by_sql(s).each do |row| + puts row.to_json + cache[row['id']] << [row['seg_distance'], row['seg_percent'], row['seg_length'], row['seg_length_sum']] + end + return cache + # Get the closest segment, weighted by how far it advances shape_dist_traveled + # shape_dist_traveled = 0.0 + # trip_pattern.each do |i| + # a = cache[i] || [] + # scored = a.map do |seg_distance, seg_percent, seg_length, seg_length_sum| + # dist = (seg_length * seg_percent + seg_length_sum) - shape_dist_traveled + # score = dist / seg_distance + # [score, dist] + # end + # s = scored.select { |_,seg_distance| seg_distance >= shape_dist_traveled }.sort.last + # if s + # puts "#{i} -> #{s[1]}" + # else + # puts "#{i} -> no result" + # end + # end + # return cache + end + + def self.get_linear_stop_distances(trip_pattern, distances=nil) + distances ||= {} + return distances unless trip_pattern.size > 0 + trip_pattern = trip_pattern.map(&:to_i) + # Raw SQL, but difficult to get cte's otherwise. + s = <<-EOF + WITH + gtfs_stops AS ( + SELECT id, geometry::geometry + FROM gtfs_stops + INNER JOIN ( + SELECT unnest, ordinality + FROM unnest( ARRAY[#{trip_pattern.join(',')}] ) WITH ORDINALITY + ) as unnest + ON gtfs_stops.id = unnest + ORDER BY ordinality + ), + shapes AS ( + SELECT ST_Length(ST_MakeLine(geometry)::geography) AS shape_length, ST_MakeLine(geometry) AS geometry FROM gtfs_stops + ) + SELECT + gtfs_stops.id, + shapes.shape_length, + ST_LineLocatePoint(shapes.geometry::geometry, gtfs_stops.geometry::geometry) AS shape_percent + FROM gtfs_stops + INNER JOIN shapes ON true + EOF + GTFSStop.find_by_sql(s.squish).each do |row| + distances[nil] ||= row.shape_length + distances[row.id] = row.shape_percent + end + return distances + end + + def self.get_shape_stop_distances(trip_pattern, shape_id, distances=nil) + # Calculate line percent from closest point to stop + distances ||= {} + return distances unless trip_pattern.size > 0 + shape_id = shape_id.to_i + trip_pattern = trip_pattern.map(&:to_i) + # Raw SQL, as above. + s = <<-EOF + WITH + shapes AS ( + SELECT + gtfs_shapes.id, + gtfs_shapes.geometry, + ST_Length(gtfs_shapes.geometry) as shape_length + FROM gtfs_shapes + WHERE gtfs_shapes.id = #{shape_id} + ) + SELECT + gtfs_stops.id, + shapes.shape_length, + ST_LineLocatePoint(shapes.geometry::geometry, gtfs_stops.geometry::geometry) AS shape_percent + FROM gtfs_stops + INNER JOIN shapes ON true + WHERE gtfs_stops.id IN (#{trip_pattern.join(',')}); + EOF + GTFSStop.find_by_sql(s.squish).each do |row| + distances[nil] ||= row.shape_length + distances[row.id] = row.shape_percent + end + return distances + end end From cea5553a09064270ff392d01a382a5385175aa77 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Wed, 15 Aug 2018 01:47:52 -0700 Subject: [PATCH 10/14] WIP --- app/services/gtfs_stop_time_service.rb | 75 +++++++++----------------- 1 file changed, 25 insertions(+), 50 deletions(-) diff --git a/app/services/gtfs_stop_time_service.rb b/app/services/gtfs_stop_time_service.rb index 8cf6d0215..1f81f1cb1 100644 --- a/app/services/gtfs_stop_time_service.rb +++ b/app/services/gtfs_stop_time_service.rb @@ -136,8 +136,9 @@ def self.interpolate_find_dist(stop_times) return gaps end - def self.get_shape_stop_segments(trip_pattern, shape_id, cache=nil, maxdistance=200, segments=100) - # Calculate line percent from closest point to stop + def self.get_shape_stop_segments(trip_pattern, shape_id, cache=nil, maxdistance=200, segments=1) + # Split line into segments, + # get distance from each stop to each segment and the point on the segment. cache ||= {} trip_pattern = trip_pattern.select { |i| !cache.key?(i) } return cache unless trip_pattern.size > 0 @@ -162,18 +163,16 @@ def self.get_shape_stop_segments(trip_pattern, shape_id, cache=nil, maxdistance= linesubs AS ( SELECT i AS seg_id, - ST_LineSubstring(a.geometry, (i::float)/#{segments},(i+1::float)/#{segments})::geography AS shape, - shape_length + ST_LineSubstring(a.geometry, i/#{segments}::float,(i+1)/#{segments}::float)::geography AS shape FROM a, GENERATE_SERIES(0,#{segments-1}) AS i ), segs AS ( SELECT seg_id, shape, - shape_length, - ST_Length(shape) as seg_length, - sum(ST_Length(shape)) OVER (ORDER BY seg_id) AS seg_length_sum - FROM linesubs + ST_Length(shape::geography) as seg_length, + sum(ST_Length(shape::geography)) OVER (ORDER BY seg_id) AS seg_length_sum + FROM linesubs ORDER BY seg_id ), gtfs_stops AS ( SELECT @@ -185,26 +184,35 @@ def self.get_shape_stop_segments(trip_pattern, shape_id, cache=nil, maxdistance= gtfs_stops_segs AS ( SELECT gtfs_stops.id, + gtfs_stops.geometry, seg_id, - segs.seg_length, - segs.shape_length as shape_length, - (segs.seg_length_sum - segs.seg_length) AS seg_length_sum, - ST_LineLocatePoint(segs.shape::geometry, gtfs_stops.geometry::geometry) AS seg_percent, - ST_Distance(segs.shape, gtfs_stops.geometry) AS seg_distance + ST_Distance(segs.shape, gtfs_stops.geometry, false) AS seg_distance, + ST_ShortestLine(segs.shape::geometry, gtfs_stops.geometry::geometry) AS seg_shortest FROM gtfs_stops INNER JOIN segs ON true ) SELECT - * - FROM gtfs_stops_segs + id, + ST_Length(seg_shortest::geography, false) AS seg_distance, + seg_id, + segs.seg_length_sum - segs.seg_length + ST_Length(ST_LineSubstring(segs.shape::geometry, 0.0, ST_LineLocatePoint(segs.shape::geometry, gtfs_stops_segs.geometry::geometry))::geography) as seg_traveled + FROM gtfs_stops_segs INNER JOIN segs USING(seg_id) WHERE seg_distance < #{maxdistance} EOF - # s = s.squish + s = s.squish trip_pattern.each { |i| cache[i] ||= [] } GTFSStop.find_by_sql(s).each do |row| puts row.to_json - cache[row['id']] << [row['seg_distance'], row['seg_percent'], row['seg_length'], row['seg_length_sum']] + cache[row['id']] << [row['seg_distance'], row['seg_traveled']] end + trip_pattern.each do |i| + puts "#{i} -> #{cache[i].sort.first}" + end + + + + + return cache # Get the closest segment, weighted by how far it advances shape_dist_traveled # shape_dist_traveled = 0.0 @@ -229,7 +237,6 @@ def self.get_linear_stop_distances(trip_pattern, distances=nil) distances ||= {} return distances unless trip_pattern.size > 0 trip_pattern = trip_pattern.map(&:to_i) - # Raw SQL, but difficult to get cte's otherwise. s = <<-EOF WITH gtfs_stops AS ( @@ -258,36 +265,4 @@ def self.get_linear_stop_distances(trip_pattern, distances=nil) end return distances end - - def self.get_shape_stop_distances(trip_pattern, shape_id, distances=nil) - # Calculate line percent from closest point to stop - distances ||= {} - return distances unless trip_pattern.size > 0 - shape_id = shape_id.to_i - trip_pattern = trip_pattern.map(&:to_i) - # Raw SQL, as above. - s = <<-EOF - WITH - shapes AS ( - SELECT - gtfs_shapes.id, - gtfs_shapes.geometry, - ST_Length(gtfs_shapes.geometry) as shape_length - FROM gtfs_shapes - WHERE gtfs_shapes.id = #{shape_id} - ) - SELECT - gtfs_stops.id, - shapes.shape_length, - ST_LineLocatePoint(shapes.geometry::geometry, gtfs_stops.geometry::geometry) AS shape_percent - FROM gtfs_stops - INNER JOIN shapes ON true - WHERE gtfs_stops.id IN (#{trip_pattern.join(',')}); - EOF - GTFSStop.find_by_sql(s.squish).each do |row| - distances[nil] ||= row.shape_length - distances[row.id] = row.shape_percent - end - return distances - end end From 76051722a701eb8f49f35959dc3258dba04b66d1 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Wed, 15 Aug 2018 03:03:58 -0700 Subject: [PATCH 11/14] WIP --- app/services/gtfs_stop_time_service.rb | 64 ++++++++++++-------------- 1 file changed, 29 insertions(+), 35 deletions(-) diff --git a/app/services/gtfs_stop_time_service.rb b/app/services/gtfs_stop_time_service.rb index 1f81f1cb1..285db1dfb 100644 --- a/app/services/gtfs_stop_time_service.rb +++ b/app/services/gtfs_stop_time_service.rb @@ -146,73 +146,67 @@ def self.get_shape_stop_segments(trip_pattern, shape_id, cache=nil, maxdistance= trip_pattern = trip_pattern.map(&:to_i) maxdistance = maxdistance.to_i segments = segments.to_i - # a: get the shape, as geometry - # linesubs: break shape into 100 segments, as geography + # linesubs: break shape into segments, as geography # segs: get cumulative line distance, including the current segment # gtfs_stops: get stops # gtfs_stops_segs: find distance between each stop and each segment + # filter by stop distance from line < maxdistance s = <<-EOF WITH - a AS ( - SELECT - gtfs_shapes.id, - gtfs_shapes.geometry::geometry as geometry, - ST_Length(gtfs_shapes.geometry) as shape_length - FROM gtfs_shapes WHERE id = #{shape_id} - ), linesubs AS ( SELECT i AS seg_id, - ST_LineSubstring(a.geometry, i/#{segments}::float,(i+1)/#{segments}::float)::geography AS shape - FROM a, GENERATE_SERIES(0,#{segments-1}) AS i + ST_LineSubstring(gtfs_shapes.geometry::geometry, i/#{segments}::float,(i+1)/#{segments}::float)::geography AS shape + FROM gtfs_shapes, GENERATE_SERIES(0,#{segments-1}) AS i + WHERE id = #{shape_id} ), segs AS ( SELECT seg_id, shape, - ST_Length(shape::geography) as seg_length, - sum(ST_Length(shape::geography)) OVER (ORDER BY seg_id) AS seg_length_sum - FROM linesubs ORDER BY seg_id - ), - gtfs_stops AS ( - SELECT - gtfs_stops.id, - gtfs_stops.geometry - FROM gtfs_stops - WHERE gtfs_stops.id IN (#{trip_pattern.join(',')}) + ST_Length(shape) as seg_length, + sum(ST_Length(shape)) OVER (ORDER BY seg_id) AS seg_length_sum + FROM linesubs ), gtfs_stops_segs AS ( SELECT gtfs_stops.id, - gtfs_stops.geometry, seg_id, - ST_Distance(segs.shape, gtfs_stops.geometry, false) AS seg_distance, - ST_ShortestLine(segs.shape::geometry, gtfs_stops.geometry::geometry) AS seg_shortest + ST_ShortestLine(segs.shape::geometry, gtfs_stops.geometry::geometry) AS seg_shortest, + ST_Length(ST_ShortestLine(segs.shape::geometry, gtfs_stops.geometry::geometry)::geography) AS seg_distance FROM gtfs_stops INNER JOIN segs ON true + WHERE gtfs_stops.id IN (#{trip_pattern.join(',')}) ) SELECT id, - ST_Length(seg_shortest::geography, false) AS seg_distance, seg_id, - segs.seg_length_sum - segs.seg_length + ST_Length(ST_LineSubstring(segs.shape::geometry, 0.0, ST_LineLocatePoint(segs.shape::geometry, gtfs_stops_segs.geometry::geometry))::geography) as seg_traveled - FROM gtfs_stops_segs INNER JOIN segs USING(seg_id) + seg_distance, + ST_AsGeoJSON(seg_shortest) AS seg_shortest_geojson, + ST_Length( + ST_LineSubstring( + segs.shape::geometry, + 0.0, + ST_LineLocatePoint( + segs.shape::geometry, + ST_PointN(seg_shortest, 1) + ) + )::geography + ) + segs.seg_length_sum - segs.seg_length AS seg_traveled + FROM gtfs_stops_segs INNER JOIN segs USING(seg_id) WHERE seg_distance < #{maxdistance} EOF s = s.squish trip_pattern.each { |i| cache[i] ||= [] } + features = [] GTFSStop.find_by_sql(s).each do |row| puts row.to_json cache[row['id']] << [row['seg_distance'], row['seg_traveled']] + features << {type: 'Feature', properties: {"stroke" => "#ff0018", "stroke-width": 4}, geometry: JSON.parse(row['seg_shortest_geojson'])} end - trip_pattern.each do |i| - puts "#{i} -> #{cache[i].sort.first}" - end - - - - - + GTFSStop.find(trip_pattern).each { |s| features << {type: 'Feature', properties: {}, geometry: s.geometry(as: :geojson) }} + features << {type: 'Feature', properties: {"stroke-width": 4, "stroke" => "#0000ff"}, geometry: GTFSShape.find(shape_id).geometry(as: :geojson)} + puts ({type: "FeatureCollection", features: features}).to_json return cache # Get the closest segment, weighted by how far it advances shape_dist_traveled # shape_dist_traveled = 0.0 From 339aa3c95cf7c24c3e12b7e354184f3b8dce7d9d Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Wed, 15 Aug 2018 03:48:31 -0700 Subject: [PATCH 12/14] WIP --- app/services/gtfs_stop_time_service.rb | 40 +++++++++++--------------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/app/services/gtfs_stop_time_service.rb b/app/services/gtfs_stop_time_service.rb index 285db1dfb..1224ce6d8 100644 --- a/app/services/gtfs_stop_time_service.rb +++ b/app/services/gtfs_stop_time_service.rb @@ -228,35 +228,27 @@ def self.get_shape_stop_segments(trip_pattern, shape_id, cache=nil, maxdistance= end def self.get_linear_stop_distances(trip_pattern, distances=nil) - distances ||= {} - return distances unless trip_pattern.size > 0 trip_pattern = trip_pattern.map(&:to_i) + t1 = trip_pattern[0..-2] + t2 = trip_pattern[1..-1] s = <<-EOF - WITH - gtfs_stops AS ( - SELECT id, geometry::geometry - FROM gtfs_stops - INNER JOIN ( - SELECT unnest, ordinality - FROM unnest( ARRAY[#{trip_pattern.join(',')}] ) WITH ORDINALITY - ) as unnest - ON gtfs_stops.id = unnest - ORDER BY ordinality - ), - shapes AS ( - SELECT ST_Length(ST_MakeLine(geometry)::geography) AS shape_length, ST_MakeLine(geometry) AS geometry FROM gtfs_stops - ) SELECT - gtfs_stops.id, - shapes.shape_length, - ST_LineLocatePoint(shapes.geometry::geometry, gtfs_stops.geometry::geometry) AS shape_percent - FROM gtfs_stops - INNER JOIN shapes ON true + o.id AS id, + o.seg_id AS seg_id, + ST_Length(ST_MakeLine(o.geometry::geometry, d.geometry::geometry)::geography) AS seg_length + FROM + (SELECT stop_name,seg_id,id,geometry FROM gtfs_stops, unnest(array[#{t1.join(',')}]) WITH ORDINALITY AS t(unnest,seg_id) where id = unnest) o, + (SELECT stop_name,seg_id,id,geometry FROM gtfs_stops, unnest(array[#{t2.join(',')}]) WITH ORDINALITY AS t(unnest,seg_id) where id = unnest) d + WHERE o.seg_id = d.seg_id ORDER BY o.seg_id EOF + cache = {} + d = 0.0 GTFSStop.find_by_sql(s.squish).each do |row| - distances[nil] ||= row.shape_length - distances[row.id] = row.shape_percent + puts row.to_json + d += row['seg_length'] + cache[row['id']] = [0.0, d] end - return distances + puts trip_pattern.map { |i| cache[i] } + return cache end end From 2b76bac6e340c82f31b00f200016625cf488c620 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Tue, 28 Aug 2018 11:52:48 -0700 Subject: [PATCH 13/14] Shuffle files --- app/services/tile_export_gtfs_service.rb | 561 +++++++++++++++++++++++ app/services/tile_export_service.rb | 284 ++++++------ 2 files changed, 701 insertions(+), 144 deletions(-) create mode 100644 app/services/tile_export_gtfs_service.rb diff --git a/app/services/tile_export_gtfs_service.rb b/app/services/tile_export_gtfs_service.rb new file mode 100644 index 000000000..0edf62f42 --- /dev/null +++ b/app/services/tile_export_gtfs_service.rb @@ -0,0 +1,561 @@ +module TileExportGTFSService + BBOX_PADDING = 0.1 + KEY_QUEUE_STOPS = 'queue_stops' + KEY_QUEUE_SCHEDULES = 'queue_schedules' + KEY_STOPID_GRAPHID = 'stopid_graphid' + IMPORT_LEVEL = 4 + GRAPH_LEVEL = 2 + STOP_PAIRS_TILE_LIMIT = 500_000 + + # kTransitEgress = 4, // Transit egress + # kTransitStation = 5, // Transit station + # kMultiUseTransitPlatform = 6, // Multi-use transit platform (rail and bus) + NODE_TYPES = { + 2 => 4, + 1 => 5, + 0 => 6 + } + + VT = Valhalla::Mjolnir::Transit::VehicleType + VEHICLE_TYPES = { + tram: VT::Tram, + tram_service: VT::Tram, + metro: VT::Metro, + rail: VT::Rail, + suburban_railway: VT::Rail, + bus: VT::Bus, + trolleybys_service: VT::Bus, + express_bus_service: VT::Bus, + local_bus_service: VT::Bus, + bus_service: VT::Bus, + shuttle_bus: VT::Bus, + demand_and_response_bus_service: VT::Bus, + regional_bus_service: VT::Bus, + ferry: VT::Ferry, + cablecar: VT::CableCar, + gondola: VT::Gondola, + funicular: VT::Funicular + } + + class TileValueError < StandardError + end + + class OriginEqualsDestinationError < TileValueError + end + + class MissingGraphIDError < TileValueError + end + + class MissingRouteError < TileValueError + end + + class MissingShapeError < TileValueError + end + + class MissingTripError < TileValueError + end + + class InvalidTimeError < TileValueError + end + + class TileBuilder + attr_accessor :tile + def initialize(tilepath, tile, feed_version_ids: nil) + @tilepath = tilepath + @tile = tile + # filters + @feed_version_ids = feed_version_ids || [] + # globally unique indexes + @stopid_graphid ||= {} + @graphid_stopid ||= {} + @trip_index ||= TileUtils::DigestIndex.new(bits: 24) + @block_index ||= TileUtils::DigestIndex.new(start: 1, bits: 20) + # tile unique indexes + @route_index = {} + @shape_index = {} + end + + def log(msg) + puts "tile #{@tile}: #{msg}" + end + + def debug(msg) + puts "tile #{@tile} debug: #{msg}" + end + + def build_stops + # TODO: + # max graph_ids in a tile + t = Time.now + + # New tile + tileset = TileUtils::TileSet.new(@tilepath) + tile = tileset.new_tile(GRAPH_LEVEL, @tile) + + GTFSStop + .where(feed_version_id: @feed_version_ids) + .where(parent_station_id: nil) + .geometry_within_bbox(bbox_padded(tile.bbox)) + .includes(:children) + .find_each do |stop| + + # Check if stop is inside tile + stop_tile = TileUtils::GraphID.new(level: GRAPH_LEVEL, lon: stop.stop_lon, lat: stop.stop_lat).tile + if stop_tile != @tile + # debug("skipping stop #{stop.id}: coordinates #{stop.coordinates.join(',')} map to tile #{stop_tile} outside of tile #{@tile}") + next + end + + # Station references + prev_type_graphid = nil + + # Egresses + stop_egresses = [] # stop.stop_egresses.to_a + (stop_egresses << GTFSStop.new(stop.attributes)) if stop_egresses.empty? # generated egress + stop_egresses.each do |stop_egress| + stop_egress.location_type = 2 + node = make_node(stop_egress) + node.graphid = TileUtils::GraphID.new(level: GRAPH_LEVEL, tile: @tile, index: tile.message.nodes.size).value + node.prev_type_graphid = prev_type_graphid if prev_type_graphid + prev_type_graphid ||= node.graphid + tile.message.nodes << node + end + + # Station + stop.location_type = 1 + node = make_node(stop) + node.graphid = TileUtils::GraphID.new(level: GRAPH_LEVEL, tile: @tile, index: tile.message.nodes.size).value + node.prev_type_graphid = prev_type_graphid if prev_type_graphid + prev_type_graphid = node.graphid + tile.message.nodes << node + + # Platforms + stop_platforms = stop.children.to_a + (stop_platforms << GTFSStop.new(stop.attributes)) if stop_platforms.empty? + stop_platforms.each do |stop_platform| + stop_platform.location_type = 0 + node = make_node(stop_platform) + node.graphid = TileUtils::GraphID.new(level: GRAPH_LEVEL, tile: @tile, index: tile.message.nodes.size).value + node.prev_type_graphid = prev_type_graphid if prev_type_graphid # station_id graphid + @stopid_graphid[stop_platform.id] = node.graphid + @graphid_stopid[node.graphid] = stop_platform.id + tile.message.nodes << node + end + end + + # Write tile + nodes_size = tile.message.nodes.size + tileset.write_tile(tile) if nodes_size > 0 + t = Time.now - t + log("nodes: #{nodes_size} time: #{t.round(2)} (#{(nodes_size/t).to_i} nodes/s)") + return nodes_size + end + + def build_schedules + # Get stop_ids + t = Time.now + tileset = TileUtils::TileSet.new(@tilepath) + base_tile = tileset.read_tile(GRAPH_LEVEL, @tile) + stop_ids = base_tile.message.nodes.map { |node| @graphid_stopid[node.graphid] }.compact + trips = {} + calendars = {} + + # Build stop_pairs for each stop_id + tile_ext = 0 + stop_pairs_tile = base_tile # tileset.new_tile(GRAPH_LEVEL, @tile) + stop_pairs_total = 0 + errors = Hash.new(0) + stop_ids.each do |stop_id| + puts "stop_id: #{stop_id}" + stop_pairs_stop_id_count = 0 + GTFSStopTime + .where(stop_id: stop_id) + .find_in_batches do |ssps| + + # Skip the last stop_time in a trip + ssps = ssps.select(&:destination_id) + + # Cache: Get unseen trips + trip_ids = ssps.map(&:trip_id).select { |t| !trips.key?(t) } + GTFSTrip.find(trip_ids).each do |t| + trips[t.id] = t + end + ssps.each { |ssp| ssp.trip = trips[ssp.trip_id] } + + # Cache: Get unseen calendars and calendar_dates + ssps.each do |ssp| + # if we don't have the calendar cached, find the calendar or create an empty one + args = {feed_version_id: ssp.trip.feed_version_id, service_id: ssp.trip.service_id} + key = [ssp.trip.feed_version_id, ssp.trip.service_id] + calendar = calendars[key] + if calendar.nil? + calendar = GTFSCalendar.find_by(args) # || GTFSCalendar.new(args) + end + calendar.start_date ||= calendar.service_added_dates.min + calendar.end_date ||= calendar.service_added_dates.max + ssp.trip.calendar = calendar + calendars[key] = calendar + end + + # Get unvisited routes for this batch, add to tile + route_ids = ssps.map { |ssp| ssp.trip.route_id }.select { |route_id| !@route_index.key?(route_id) }.uniq + ssps.each { |ssp| @route_index[ssp.trip.route_id] ||= nil } # set as visited + if route_ids.size > 0 + GTFSRoute.where(id: route_ids).find_each do |route| # get unvisited + @route_index[route.id] = base_tile.message.routes.size + debug("route: #{route.id} -> #{@route_index[route.id]}") + base_tile.message.routes << make_route(route) + end + end + + # Get unseen shapes for this batch, add to tile + shape_ids = ssps.map { |ssp| ssp.trip.shape_id }.select { |shape_id| !@shape_index.key?(shape_id) }.uniq + ssps.each { |ssp| @shape_index[ssp.trip.shape_id] ||= nil } # set as visited + if shape_ids.size > 0 + GTFSShape.where(id: shape_ids).find_each do |shape| + pshape = make_shape(shape) + pshape.shape_id = base_tile.message.shapes.size + 1 + @shape_index[shape.id] = pshape.shape_id + debug("shape: #{shape.id} -> #{@shape_index[shape.id]}") + base_tile.message.shapes << pshape + end + end + + # Process each ssp + ssps.each do |ssp| + # process ssp and count errors + begin + i = make_stop_pair(ssp) + rescue TileValueError => e + errors[e.class.name.to_sym] += 1 + log("error: ssp #{ssp.id}: #{e}") + rescue StandardError => e + errors[e.class.name.to_sym] += 1 + log("error: ssp #{ssp.id}: #{e}") + end + next unless i + stop_pairs_tile.message.stop_pairs << i + stop_pairs_stop_id_count += 1 + stop_pairs_total += 1 + end + + # Write supplement tile, start new tile + if stop_pairs_tile.message.stop_pairs.size > STOP_PAIRS_TILE_LIMIT + if stop_pairs_tile != base_tile + debug("writing tile ext #{tile_ext}: #{stop_pairs_tile.message.stop_pairs.size} stop_pairs") + tileset.write_tile(stop_pairs_tile, ext: tile_ext) + tile_ext += 1 + end + stop_pairs_tile = tileset.new_tile(GRAPH_LEVEL, @tile) + end + end + # Done for this stop + debug("stop_pairs for stop_id #{stop_id}: #{stop_pairs_stop_id_count}") + end + + # Write dangling supplement tile + if stop_pairs_tile != base_tile && stop_pairs_tile.message.stop_pairs.size > 0 + debug("writing tile ext #{tile_ext}: #{stop_pairs_tile.message.stop_pairs.size} stop_pairs") + tileset.write_tile(stop_pairs_tile, ext: tile_ext) + end + + # Write the base tile + debug("writing tile base: #{base_tile.message.nodes.size} nodes, #{base_tile.message.routes.size} routes, #{base_tile.message.shapes.size} shapes, #{base_tile.message.stop_pairs.size} stop_pairs (#{stop_pairs_total} tile total)") + tileset.write_tile(base_tile) + + # Write tile + t = Time.now - t + error_txt = ([errors.values.sum.to_s] + errors.map { |k,v| "#{k}: #{v}" }).join(' ') + log("#{base_tile.message.nodes.size} nodes, #{base_tile.message.routes.size} routes, #{base_tile.message.shapes.size} shapes, #{stop_pairs_total} stop_pairs, errors #{error_txt}, time: #{t.round(2)} (#{(stop_pairs_total/t).to_i} stop_pairs/s)") + return stop_pairs_total + end + + private + + def seconds_since_midnight(value) + h,m,s = value.split(':').map(&:to_i) + h * 3600 + m * 60 + s + end + + def color_to_int(value) + match = /(\h{6})/.match(value.to_s) + match ? match[0].to_i(16) : nil + end + + # bbox padding + def bbox_padded(bbox) + ymin, xmin, ymax, xmax = bbox + padding = BBOX_PADDING + [ymin-padding, xmin-padding, ymax+padding, xmax+padding] + end + + # make entity methods + def make_stop_pair(ssp) + # TODO: + # skip if origin_departure_time < frequency_start_time + # skip if bad time information + # add < and > to onestop_ids + trip = ssp.trip + calendar = ssp.trip.calendar + destination_graphid = @stopid_graphid[ssp.destination_id] + origin_graphid = @stopid_graphid[ssp.stop_id] + route_index = @route_index[trip.route_id] + shape_id = @shape_index[trip.shape_id] + + fail InvalidTimeError.new("missing calendar for trip #{trip.trip_id}") unless calendar + fail OriginEqualsDestinationError.new("origin_graphid #{origin_graphid} == destination_graphid #{destination_graphid}") if origin_graphid == destination_graphid + fail MissingGraphIDError.new("missing origin_graphid for stop #{ssp.stop_id}") unless origin_graphid + fail MissingGraphIDError.new("missing destination_graphid for stop #{ssp.destination_id}") unless destination_graphid + fail MissingRouteError.new("missing route_index for route #{trip.route_id}") unless route_index + fail MissingShapeError.new("missing shape for shape #{trip.shape_id}") unless shape_id + fail InvalidTimeError.new("origin_departure_time #{ssp.departure_time} > destination_arrival_time #{ssp.destination_arrival_time}") if ssp.departure_time > ssp.destination_arrival_time + + # Make SSP + params = {} + # bool bikes_allowed = 1; + (params[:bikes_allowed] = true) if trip.bikes_allowed == 1 + # uint32 block_id = 2; + (params[:block_id] = @block_index.check(trip.block_id)) if trip.block_id + # uint32 destination_arrival_time = 3; + params[:destination_arrival_time] = ssp.destination_arrival_time + # uint64 destination_graphid = 4; + params[:destination_graphid] = destination_graphid + # string destination_onestop_id = 5; + # params[:destination_onestop_id] = nil # TODO + # string operated_by_onestop_id = 6; + # params[:operated_by_onestop_id] = nil # TODO + # uint32 origin_departure_time = 7; + params[:origin_departure_time] = ssp.departure_time + # uint64 origin_graphid = 8; + params[:origin_graphid] = origin_graphid + # string origin_onestop_id = 9; + # params[:origin_onestop_id] = nil # TODO + # uint32 route_index = 10; + params[:route_index] = route_index + # repeated uint32 service_added_dates = 11; + params[:service_added_dates] = calendar.service_added_dates.map(&:jd) + # repeated bool service_days_of_week = 12; + params[:service_days_of_week] = calendar.service_days_of_week + # uint32 service_end_date = 13; + params[:service_end_date] = calendar.end_date.jd + # repeated uint32 service_except_dates = 14; + params[:service_except_dates] = calendar.service_except_dates.map(&:jd) + # uint32 service_start_date = 15; + params[:service_start_date] = calendar.start_date.jd + # string trip_headsign = 16; + params[:trip_headsign] = ssp.stop_headsign || trip.trip_headsign + # uint32 trip_id = 17; + params[:trip_id] = @trip_index.check(trip.id) + # bool wheelchair_accessible = 18; + (params[:wheelchair_accessible] = true) if trip.wheelchair_accessible == 1 + # uint32 shape_id = 20; + params[:shape_id] = shape_id + # float origin_dist_traveled = 21; + (params[:origin_dist_traveled] = ssp.shape_dist_traveled) if ssp.shape_dist_traveled + # float destination_dist_traveled = 22; + # params[:destination_dist_traveled] = nil # ssp.destination_dist_traveled if ssp.destination_dist_traveled + # TODO: frequencies + # puts params + Valhalla::Mjolnir::Transit::StopPair.new(params) + end + + def make_shape(shape) + params = {} + # uint32 shape_id = 1; + # bytes encoded_shape = 2; + # reverse coordinates + reversed = shape.geometry[:coordinates].map { |a,b| [b,a] } + params[:encoded_shape] = TileUtils::Shape7.encode(reversed) + Valhalla::Mjolnir::Transit::Shape.new(params) + end + + def make_route(route) + # TODO: + # skip if unknown vehicle_type + params = {} + # string name = 1; + params[:name] = route.name + # string onestop_id = 2; + # params[:onestop_id] = nil # TODO + # string operated_by_name = 3; + params[:operated_by_name] = route.agency.agency_name + # string operated_by_onestop_id = 4; + # params[:operated_by_onestop_id] = nil # TODO + # string operated_by_website = 5; + params[:operated_by_website] = route.agency.agency_url + # uint32 route_color = 6; + params[:route_color] = color_to_int(route.route_color || 'FFFFFF') + # string route_desc = 7; + params[:route_desc] = route.route_desc + # string route_long_name = 8; + params[:route_long_name] = route.route_long_name + # uint32 route_text_color = 9; + params[:route_text_color] = color_to_int(route.route_text_color) + # VehicleType vehicle_type = 10; + params[:vehicle_type] = VT::Bus # TODO + Valhalla::Mjolnir::Transit::Route.new(params.compact) + end + + def make_node(stop) + params = {} + # float lon = 1; + params[:lon] = stop.stop_lon + # float lat = 2; + params[:lat] = stop.stop_lat + # uint32 type = 3; + params[:type] = NODE_TYPES[stop.location_type] + # uint64 graphid = 4; + # set in build_stops + # uint64 prev_type_graphid = 5; + # set in build_stops + # string name = 6; + params[:name] = stop.stop_name + # string onestop_id = 7; + # params[:onestop_id] = nil # TODO + # uint64 osm_way_id = 8; + # params[:osm_way_id] = nil # TODO + # string timezone = 9; + params[:timezone] = stop.stop_timezone || 'America/Los_Angeles' + # bool wheelchair_boarding = 10; + params[:wheelchair_boarding] = true + # bool generated = 11; + if stop.location_type == 2 + # params[:onestop_id] = "#{onestop_id}>" + params[:generated] = true + end + if stop.location_type == 0 + # params[:onestop_id] = "#{onestop_id}<" + # params[:generated] = true # not set for platforms + end + # uint32 traversability = 12; + if stop.location_type == 2 # TODO: check + params[:traversability] = 3 + end + Valhalla::Mjolnir::Transit::Node.new(params.compact) + end + end + + def self.tile_build_stops(tilepath, feed_version_ids: nil) + redis = Redis.new + while tile = redis.rpop(KEY_QUEUE_STOPS) + tile = tile.to_i + builder = TileBuilder.new(tilepath, tile, feed_version_ids: feed_version_ids) + nodes_size = builder.build_stops + if nodes_size > 0 + redis.rpush(KEY_QUEUE_SCHEDULES, tile) + stopid_graphid = builder.instance_variable_get('@stopid_graphid') + stopid_graphid.each_slice(1000) { |i| redis.hmset(KEY_STOPID_GRAPHID, i.flatten) } + end + remaining = redis.llen(KEY_QUEUE_STOPS) + puts "remaining: ~#{remaining}" + end + end + + def self.tile_build_schedules(tilepath, feed_version_ids: nil) + # stopid_graphid + stopid_graphid = {} + graphid_stopid = {} + redis = Redis.new + cursor = nil + while cursor != '0' + cursor, data = redis.hscan(KEY_STOPID_GRAPHID, cursor, count: 1_000) + data.each do |k,v| + k = k.to_i + v = v.to_i + stopid_graphid[k] = v + graphid_stopid[v] = k + end + end + # queue + while tile = redis.rpop(KEY_QUEUE_SCHEDULES) + tile = tile.to_i + builder = TileBuilder.new(tilepath, tile, feed_version_ids: feed_version_ids) + builder.instance_variable_set('@stopid_graphid', stopid_graphid) + builder.instance_variable_set('@graphid_stopid', graphid_stopid) + builder.build_schedules + remaining = redis.llen(KEY_QUEUE_SCHEDULES) + puts "remaining: ~#{remaining}" + end + end + + def self.export_tiles(tilepath, thread_count: nil, feeds: nil, feed_versions: nil, tiles: nil) + # Debug + ActiveRecord::Base.logger = Logger.new(STDOUT) + ActiveRecord::Base.logger.level = Logger::DEBUG + + # Filter by feed/feed_version + # feed_version_ids = [] + # if feed_versions + # feed_version_ids = feed_versions.map(&:id) + # elsif feeds + # feed_version_ids = feeds.map(&:active_feed_version_id) + # else + # feed_version_ids = Feed.where_active_feed_version_import_level(IMPORT_LEVEL).pluck(:active_feed_version_id) + # end + feed_version_ids = [3] # FeedVersion.pluck(:id) + + # Build bboxes + puts "Selecting tiles..." + count_stops = Set.new + stop_platforms = Hash.new { |h,k| h[k] = Set.new } + stop_egresses = Hash.new { |h,k| h[k] = Set.new } + tiles = Set.new(tiles) + if tiles.empty? + count = 1 + total = feed_version_ids.size + FeedVersion.where(id: feed_version_ids).includes(:feed).find_each do |feed_version| + feed = feed_version.feed + fvtiles = Set.new + GTFSStop.where(feed_version: feed_version).find_each do |stop| + if stop.parent_station_id.nil? + count_stops << stop.id + fvtiles << TileUtils::GraphID.new(level: GRAPH_LEVEL, lon: stop.stop_lon, lat: stop.stop_lat).tile + else + stop_platforms[stop.parent_station_id] << stop.id + end + end + puts "\t(#{count}/#{total}) #{feed_version.feed.onestop_id} #{feed_version.sha1}: #{fvtiles.size} tiles" + tiles += fvtiles + count += 1 + end + end + + # TODO: Filter stop_platforms/stop_egresses by feed_version + count_egresses = count_stops.map { |i| stop_egresses[i].empty? ? 1 : stop_egresses[i].size }.sum + count_platforms = count_stops.map { |i| stop_platforms[i].empty? ? 1 : stop_platforms[i].size }.sum + puts "Tiles to build: #{tiles.size}" + puts "Expected:" + puts "\tstops: #{count_stops.size}" + puts "\tplatforms: #{stop_platforms.map { |k,v| v.size }.sum}" + puts "\tegresses: #{stop_egresses.map { |k,v| v.size }.sum}" + puts "\tnodes: #{count_stops.size + count_egresses + count_platforms}" + puts "\tstopid-graphid: #{count_platforms}" + + # Setup queue + thread_count ||= 1 + redis = Redis.new + redis.del(KEY_QUEUE_STOPS) + redis.del(KEY_QUEUE_SCHEDULES) + redis.del(KEY_STOPID_GRAPHID) + tiles.each_slice(1000) { |i| redis.rpush(KEY_QUEUE_STOPS, i) } + + # Build stops for each tile. + puts "\n===== Stops =====\n" + workers = (0...thread_count).map do + fork { tile_build_stops(tilepath, feed_version_ids: feed_version_ids) } + end + workers.each { |pid| Process.wait(pid) } + + puts "\nStops finished. Schedule tile queue: #{redis.llen(KEY_QUEUE_SCHEDULES)} stopid-graphid mappings: #{redis.hlen(KEY_STOPID_GRAPHID)}" + + # Build schedule, routes, shapes for each tile. + puts "\n===== Routes, Shapes, StopPairs =====\n" + workers = (0...thread_count).map do + fork { tile_build_schedules(tilepath, feed_version_ids: feed_version_ids) } + end + workers.each { |pid| Process.wait(pid) } + + puts "Done!" + end + end + \ No newline at end of file diff --git a/app/services/tile_export_service.rb b/app/services/tile_export_service.rb index 08cf9f14e..2a548a184 100644 --- a/app/services/tile_export_service.rb +++ b/app/services/tile_export_service.rb @@ -11,9 +11,9 @@ module TileExportService # kTransitStation = 5, // Transit station # kMultiUseTransitPlatform = 6, // Multi-use transit platform (rail and bus) NODE_TYPES = { - 2 => 4, - 1 => 5, - 0 => 6 + StopEgress: 4, + Stop: 5, + StopPlatform: 6 } VT = Valhalla::Mjolnir::Transit::VehicleType @@ -92,15 +92,15 @@ def build_stops tileset = TileUtils::TileSet.new(@tilepath) tile = tileset.new_tile(GRAPH_LEVEL, @tile) - GTFSStop - .where(feed_version_id: @feed_version_ids) - .where(parent_station_id: nil) + Stop + .where_imported_from_feed_version(@feed_version_ids) + .where(parent_stop: nil) .geometry_within_bbox(bbox_padded(tile.bbox)) - .includes(:children) + .includes(:stop_platforms, :stop_egresses) .find_each do |stop| # Check if stop is inside tile - stop_tile = TileUtils::GraphID.new(level: GRAPH_LEVEL, lon: stop.stop_lon, lat: stop.stop_lat).tile + stop_tile = TileUtils::GraphID.new(level: GRAPH_LEVEL, lon: stop.coordinates[0], lat: stop.coordinates[1]).tile if stop_tile != @tile # debug("skipping stop #{stop.id}: coordinates #{stop.coordinates.join(',')} map to tile #{stop_tile} outside of tile #{@tile}") next @@ -110,10 +110,9 @@ def build_stops prev_type_graphid = nil # Egresses - stop_egresses = [] # stop.stop_egresses.to_a - (stop_egresses << GTFSStop.new(stop.attributes)) if stop_egresses.empty? # generated egress + stop_egresses = stop.stop_egresses.to_a + stop_egresses << StopEgress.new(stop.attributes) if stop_egresses.empty? # generated egress stop_egresses.each do |stop_egress| - stop_egress.location_type = 2 node = make_node(stop_egress) node.graphid = TileUtils::GraphID.new(level: GRAPH_LEVEL, tile: @tile, index: tile.message.nodes.size).value node.prev_type_graphid = prev_type_graphid if prev_type_graphid @@ -122,7 +121,6 @@ def build_stops end # Station - stop.location_type = 1 node = make_node(stop) node.graphid = TileUtils::GraphID.new(level: GRAPH_LEVEL, tile: @tile, index: tile.message.nodes.size).value node.prev_type_graphid = prev_type_graphid if prev_type_graphid @@ -130,10 +128,9 @@ def build_stops tile.message.nodes << node # Platforms - stop_platforms = stop.children.to_a - (stop_platforms << GTFSStop.new(stop.attributes)) if stop_platforms.empty? + stop_platforms = stop.stop_platforms.to_a + (stop_platforms << StopPlatform.new(stop.attributes)) if stop_platforms.empty? # station ssps stop_platforms.each do |stop_platform| - stop_platform.location_type = 0 node = make_node(stop_platform) node.graphid = TileUtils::GraphID.new(level: GRAPH_LEVEL, tile: @tile, index: tile.message.nodes.size).value node.prev_type_graphid = prev_type_graphid if prev_type_graphid # station_id graphid @@ -157,8 +154,6 @@ def build_schedules tileset = TileUtils::TileSet.new(@tilepath) base_tile = tileset.read_tile(GRAPH_LEVEL, @tile) stop_ids = base_tile.message.nodes.map { |node| @graphid_stopid[node.graphid] }.compact - trips = {} - calendars = {} # Build stop_pairs for each stop_id tile_ext = 0 @@ -166,66 +161,41 @@ def build_schedules stop_pairs_total = 0 errors = Hash.new(0) stop_ids.each do |stop_id| - puts "stop_id: #{stop_id}" stop_pairs_stop_id_count = 0 - GTFSStopTime - .where(stop_id: stop_id) + ScheduleStopPair + .where(origin_id: stop_id) + .includes(:origin, :destination, :operator) .find_in_batches do |ssps| - - # Skip the last stop_time in a trip - ssps = ssps.select(&:destination_id) - - # Cache: Get unseen trips - trip_ids = ssps.map(&:trip_id).select { |t| !trips.key?(t) } - GTFSTrip.find(trip_ids).each do |t| - trips[t.id] = t + # Evaluate by active feed version - faster than join or where in (?) + ssps = ssps.select { |ssp| @feed_version_ids.include?(ssp.feed_version_id) } + + # Get unvisited routes for this batch + route_ids = ssps.map(&:route_id).select { |route_id| !@route_index.key?(route_id) } + ssps.each { |ssp| @route_index[ssp.route_id] ||= nil } # set as visited + Route.where(id: route_ids).find_each do |route| # get unvisited + @route_index[route.id] = base_tile.message.routes.size + debug("route: #{route.id} -> #{@route_index[route.id]}") + base_tile.message.routes << make_route(route) end - ssps.each { |ssp| ssp.trip = trips[ssp.trip_id] } - # Cache: Get unseen calendars and calendar_dates - ssps.each do |ssp| - # if we don't have the calendar cached, find the calendar or create an empty one - args = {feed_version_id: ssp.trip.feed_version_id, service_id: ssp.trip.service_id} - key = [ssp.trip.feed_version_id, ssp.trip.service_id] - calendar = calendars[key] - if calendar.nil? - calendar = GTFSCalendar.find_by(args) # || GTFSCalendar.new(args) - end - calendar.start_date ||= calendar.service_added_dates.min - calendar.end_date ||= calendar.service_added_dates.max - ssp.trip.calendar = calendar - calendars[key] = calendar - end - - # Get unvisited routes for this batch, add to tile - route_ids = ssps.map { |ssp| ssp.trip.route_id }.select { |route_id| !@route_index.key?(route_id) }.uniq - ssps.each { |ssp| @route_index[ssp.trip.route_id] ||= nil } # set as visited - if route_ids.size > 0 - GTFSRoute.where(id: route_ids).find_each do |route| # get unvisited - @route_index[route.id] = base_tile.message.routes.size - debug("route: #{route.id} -> #{@route_index[route.id]}") - base_tile.message.routes << make_route(route) - end - end - - # Get unseen shapes for this batch, add to tile - shape_ids = ssps.map { |ssp| ssp.trip.shape_id }.select { |shape_id| !@shape_index.key?(shape_id) }.uniq - ssps.each { |ssp| @shape_index[ssp.trip.shape_id] ||= nil } # set as visited - if shape_ids.size > 0 - GTFSShape.where(id: shape_ids).find_each do |shape| - pshape = make_shape(shape) - pshape.shape_id = base_tile.message.shapes.size + 1 - @shape_index[shape.id] = pshape.shape_id - debug("shape: #{shape.id} -> #{@shape_index[shape.id]}") - base_tile.message.shapes << pshape - end + # Get unseen rsps for this batch + rsp_ids = ssps.map(&:route_stop_pattern_id).select { |rsp_id| !@shape_index.key?(rsp_id) } + ssps.each { |ssp| @shape_index[ssp.route_stop_pattern_id] ||= nil } # set as visited + RouteStopPattern.where(id: rsp_ids).find_each do |rsp| + shape = make_shape(rsp) + shape.shape_id = base_tile.message.shapes.size + 1 + @shape_index[rsp.id] = shape.shape_id + debug("shape: #{rsp.id} -> #{@shape_index[rsp.id]}") + base_tile.message.shapes << shape end # Process each ssp ssps.each do |ssp| # process ssp and count errors begin - i = make_stop_pair(ssp) + stop_pairs_tile.message.stop_pairs << make_stop_pair(ssp) + stop_pairs_stop_id_count += 1 + stop_pairs_total += 1 rescue TileValueError => e errors[e.class.name.to_sym] += 1 log("error: ssp #{ssp.id}: #{e}") @@ -233,10 +203,6 @@ def build_schedules errors[e.class.name.to_sym] += 1 log("error: ssp #{ssp.id}: #{e}") end - next unless i - stop_pairs_tile.message.stop_pairs << i - stop_pairs_stop_id_count += 1 - stop_pairs_total += 1 end # Write supplement tile, start new tile @@ -295,76 +261,86 @@ def make_stop_pair(ssp) # skip if origin_departure_time < frequency_start_time # skip if bad time information # add < and > to onestop_ids - trip = ssp.trip - calendar = ssp.trip.calendar destination_graphid = @stopid_graphid[ssp.destination_id] - origin_graphid = @stopid_graphid[ssp.stop_id] - route_index = @route_index[trip.route_id] - shape_id = @shape_index[trip.shape_id] - - fail InvalidTimeError.new("missing calendar for trip #{trip.trip_id}") unless calendar + origin_graphid = @stopid_graphid[ssp.origin_id] fail OriginEqualsDestinationError.new("origin_graphid #{origin_graphid} == destination_graphid #{destination_graphid}") if origin_graphid == destination_graphid - fail MissingGraphIDError.new("missing origin_graphid for stop #{ssp.stop_id}") unless origin_graphid + fail MissingGraphIDError.new("missing origin_graphid for stop #{ssp.origin_id}") unless origin_graphid fail MissingGraphIDError.new("missing destination_graphid for stop #{ssp.destination_id}") unless destination_graphid - fail MissingRouteError.new("missing route_index for route #{trip.route_id}") unless route_index - fail MissingShapeError.new("missing shape for shape #{trip.shape_id}") unless shape_id - fail InvalidTimeError.new("origin_departure_time #{ssp.departure_time} > destination_arrival_time #{ssp.destination_arrival_time}") if ssp.departure_time > ssp.destination_arrival_time + + route_index = @route_index[ssp.route_id] + fail MissingRouteError.new("missing route_index for route #{ssp.route_id}") unless route_index + + shape_id = @shape_index[ssp.route_stop_pattern_id] + fail MissingShapeError.new("missing shape for rsp #{ssp.route_stop_pattern_id}") unless shape_id + + trip_id = @trip_index.check(ssp.trip) + fail MissingTripError.new("missing trip_id for trip #{ssp.trip}") unless trip_id + + destination_arrival_time = seconds_since_midnight(ssp.destination_arrival_time) + origin_departure_time = seconds_since_midnight(ssp.origin_departure_time) + fail InvalidTimeError.new("origin_departure_time #{origin_departure_time} > destination_arrival_time #{destination_arrival_time}") if origin_departure_time > destination_arrival_time + + block_id = @block_index.check(ssp.block_id) # Make SSP params = {} # bool bikes_allowed = 1; - (params[:bikes_allowed] = true) if trip.bikes_allowed == 1 # uint32 block_id = 2; - (params[:block_id] = @block_index.check(trip.block_id)) if trip.block_id + # params[:block_id] = block_id # uint32 destination_arrival_time = 3; - params[:destination_arrival_time] = ssp.destination_arrival_time + params[:destination_arrival_time] = destination_arrival_time # uint64 destination_graphid = 4; params[:destination_graphid] = destination_graphid # string destination_onestop_id = 5; - # params[:destination_onestop_id] = nil # TODO + params[:destination_onestop_id] = ssp.destination.onestop_id # string operated_by_onestop_id = 6; - # params[:operated_by_onestop_id] = nil # TODO + params[:operated_by_onestop_id] = ssp.operator.onestop_id # uint32 origin_departure_time = 7; - params[:origin_departure_time] = ssp.departure_time + params[:origin_departure_time] = origin_departure_time # uint64 origin_graphid = 8; params[:origin_graphid] = origin_graphid # string origin_onestop_id = 9; - # params[:origin_onestop_id] = nil # TODO + params[:origin_onestop_id] = ssp.origin.onestop_id # uint32 route_index = 10; params[:route_index] = route_index # repeated uint32 service_added_dates = 11; - params[:service_added_dates] = calendar.service_added_dates.map(&:jd) + params[:service_added_dates] = ssp.service_added_dates.map(&:jd) # repeated bool service_days_of_week = 12; - params[:service_days_of_week] = calendar.service_days_of_week + params[:service_days_of_week] = ssp.service_days_of_week # uint32 service_end_date = 13; - params[:service_end_date] = calendar.end_date.jd + params[:service_end_date] = ssp.service_end_date.jd # repeated uint32 service_except_dates = 14; - params[:service_except_dates] = calendar.service_except_dates.map(&:jd) + params[:service_except_dates] = ssp.service_except_dates.map(&:jd) # uint32 service_start_date = 15; - params[:service_start_date] = calendar.start_date.jd + params[:service_start_date] = ssp.service_start_date.jd # string trip_headsign = 16; - params[:trip_headsign] = ssp.stop_headsign || trip.trip_headsign + params[:trip_headsign] = ssp.trip_headsign # uint32 trip_id = 17; - params[:trip_id] = @trip_index.check(trip.id) + params[:trip_id] = trip_id # bool wheelchair_accessible = 18; - (params[:wheelchair_accessible] = true) if trip.wheelchair_accessible == 1 + params[:wheelchair_accessible] = true # !!(ssp.wheelchair_accessible) # uint32 shape_id = 20; params[:shape_id] = shape_id # float origin_dist_traveled = 21; - (params[:origin_dist_traveled] = ssp.shape_dist_traveled) if ssp.shape_dist_traveled + params[:origin_dist_traveled] = ssp.origin_dist_traveled if ssp.origin_dist_traveled # float destination_dist_traveled = 22; - # params[:destination_dist_traveled] = nil # ssp.destination_dist_traveled if ssp.destination_dist_traveled - # TODO: frequencies - # puts params + params[:destination_dist_traveled] = ssp.destination_dist_traveled if ssp.destination_dist_traveled + if ssp.frequency_headway_seconds + # protobuf doesn't define frequency_start_time + # uint32 frequency_end_time = 23; + params[:frequency_end_time] = seconds_since_midnight(ssp.frequency_end_time) + # uint32 frequency_headway_seconds = 24; + params[:frequency_headway_seconds] = ssp.frequency_headway_seconds + end Valhalla::Mjolnir::Transit::StopPair.new(params) end - def make_shape(shape) + def make_shape(rsp) params = {} # uint32 shape_id = 1; # bytes encoded_shape = 2; # reverse coordinates - reversed = shape.geometry[:coordinates].map { |a,b| [b,a] } + reversed = rsp.geometry[:coordinates].map { |a,b| [b,a] } params[:encoded_shape] = TileUtils::Shape7.encode(reversed) Valhalla::Mjolnir::Transit::Shape.new(params) end @@ -376,60 +352,60 @@ def make_route(route) # string name = 1; params[:name] = route.name # string onestop_id = 2; - # params[:onestop_id] = nil # TODO + params[:onestop_id] = route.onestop_id # string operated_by_name = 3; - params[:operated_by_name] = route.agency.agency_name + params[:operated_by_name] = route.operator.name # string operated_by_onestop_id = 4; - # params[:operated_by_onestop_id] = nil # TODO + params[:operated_by_onestop_id] = route.operator.onestop_id # string operated_by_website = 5; - params[:operated_by_website] = route.agency.agency_url + params[:operated_by_website] = route.operator.website # uint32 route_color = 6; - params[:route_color] = color_to_int(route.route_color || 'FFFFFF') + params[:route_color] = color_to_int(route.color || 'FFFFFF') # string route_desc = 7; - params[:route_desc] = route.route_desc + params[:route_desc] = route.tags["route_desc"] # string route_long_name = 8; - params[:route_long_name] = route.route_long_name + params[:route_long_name] = route.tags["route_long_name"] || route.name # uint32 route_text_color = 9; - params[:route_text_color] = color_to_int(route.route_text_color) + params[:route_text_color] = color_to_int(route.tags["route_text_color"]) # VehicleType vehicle_type = 10; - params[:vehicle_type] = VT::Bus # TODO + params[:vehicle_type] = VEHICLE_TYPES[route.vehicle_type.to_sym] || VT::Bus Valhalla::Mjolnir::Transit::Route.new(params.compact) end def make_node(stop) params = {} # float lon = 1; - params[:lon] = stop.stop_lon + params[:lon] = stop.coordinates[0] # float lat = 2; - params[:lat] = stop.stop_lat + params[:lat] = stop.coordinates[1] # uint32 type = 3; - params[:type] = NODE_TYPES[stop.location_type] + params[:type] = NODE_TYPES[stop.class.name.to_sym] # uint64 graphid = 4; # set in build_stops # uint64 prev_type_graphid = 5; # set in build_stops # string name = 6; - params[:name] = stop.stop_name + params[:name] = stop.name # string onestop_id = 7; - # params[:onestop_id] = nil # TODO + params[:onestop_id] = stop.onestop_id # uint64 osm_way_id = 8; - # params[:osm_way_id] = nil # TODO + params[:osm_way_id] = stop.osm_way_id # string timezone = 9; - params[:timezone] = stop.stop_timezone || 'America/Los_Angeles' + params[:timezone] = stop.timezone # bool wheelchair_boarding = 10; params[:wheelchair_boarding] = true # bool generated = 11; - if stop.location_type == 2 - # params[:onestop_id] = "#{onestop_id}>" + if stop.instance_of?(StopEgress) && !stop.persisted? + params[:onestop_id] = "#{stop.onestop_id}>" params[:generated] = true end - if stop.location_type == 0 - # params[:onestop_id] = "#{onestop_id}<" + if stop.instance_of?(StopPlatform) && !stop.persisted? + params[:onestop_id] = "#{stop.onestop_id}<" # params[:generated] = true # not set for platforms end # uint32 traversability = 12; - if stop.location_type == 2 # TODO: check - params[:traversability] = 3 + if stop.instance_of?(StopEgress) + params[:traversability] = 3 end Valhalla::Mjolnir::Transit::Node.new(params.compact) end @@ -480,19 +456,28 @@ def self.tile_build_schedules(tilepath, feed_version_ids: nil) def self.export_tiles(tilepath, thread_count: nil, feeds: nil, feed_versions: nil, tiles: nil) # Debug - ActiveRecord::Base.logger = Logger.new(STDOUT) - ActiveRecord::Base.logger.level = Logger::DEBUG + # ActiveRecord::Base.logger = Logger.new(STDOUT) + # ActiveRecord::Base.logger.level = Logger::DEBUG + + # Avoid autoload issues in threads + Stop.connection + StopPlatform.connection + StopEgress.connection + Route.connection + Operator.connection + RouteStopPattern.connection + EntityImportedFromFeed.connection + ScheduleStopPair.connection # Filter by feed/feed_version - # feed_version_ids = [] - # if feed_versions - # feed_version_ids = feed_versions.map(&:id) - # elsif feeds - # feed_version_ids = feeds.map(&:active_feed_version_id) - # else - # feed_version_ids = Feed.where_active_feed_version_import_level(IMPORT_LEVEL).pluck(:active_feed_version_id) - # end - feed_version_ids = [3] # FeedVersion.pluck(:id) + feed_version_ids = [] + if feed_versions + feed_version_ids = feed_versions.map(&:id) + elsif feeds + feed_version_ids = feeds.map(&:active_feed_version_id) + else + feed_version_ids = Feed.where_active_feed_version_import_level(IMPORT_LEVEL).pluck(:active_feed_version_id) + end # Build bboxes puts "Selecting tiles..." @@ -506,13 +491,15 @@ def self.export_tiles(tilepath, thread_count: nil, feeds: nil, feed_versions: ni FeedVersion.where(id: feed_version_ids).includes(:feed).find_each do |feed_version| feed = feed_version.feed fvtiles = Set.new - GTFSStop.where(feed_version: feed_version).find_each do |stop| - if stop.parent_station_id.nil? - count_stops << stop.id - fvtiles << TileUtils::GraphID.new(level: GRAPH_LEVEL, lon: stop.stop_lon, lat: stop.stop_lat).tile - else - stop_platforms[stop.parent_station_id] << stop.id - end + Stop.where_imported_from_feed_version(feed_version).find_each do |stop| + if stop.is_a?(StopPlatform) + stop_platforms[stop.parent_stop_id] << stop.id + elsif stop.is_a?(StopEgress) + stop_egresses[stop.parent_stop_id] << stop.id + else + count_stops << stop.id + fvtiles << TileUtils::GraphID.new(level: GRAPH_LEVEL, lon: stop.coordinates[0], lat: stop.coordinates[1]).tile + end end puts "\t(#{count}/#{total}) #{feed_version.feed.onestop_id} #{feed_version.sha1}: #{fvtiles.size} tiles" tiles += fvtiles @@ -531,6 +518,15 @@ def self.export_tiles(tilepath, thread_count: nil, feeds: nil, feed_versions: ni puts "\tnodes: #{count_stops.size + count_egresses + count_platforms}" puts "\tstopid-graphid: #{count_platforms}" + # Clear + count_stops.clear + stop_platforms.clear + stop_egresses.clear + # stopid_graphid = Hash[redis.hgetall('stopid_graphid').map { |k,v| [k.to_i, v.to_i] }] + # expected_stops = Set.new + # count_stops.each { |i| expected_stops += (stop_platforms[i].empty? ? [i].to_set : stop_platforms[i]) } + # missing = stopid_graphid.keys.to_set - expected_stops + # Setup queue thread_count ||= 1 redis = Redis.new @@ -557,4 +553,4 @@ def self.export_tiles(tilepath, thread_count: nil, feeds: nil, feed_versions: ni puts "Done!" end -end +end \ No newline at end of file From b36308851d24b31fc5f010506c1bfe8e0c59e57b Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Tue, 28 Aug 2018 11:54:18 -0700 Subject: [PATCH 14/14] Remove --- app/services/tile_export_service_old.rb | 556 ------------------------ 1 file changed, 556 deletions(-) delete mode 100644 app/services/tile_export_service_old.rb diff --git a/app/services/tile_export_service_old.rb b/app/services/tile_export_service_old.rb deleted file mode 100644 index e62e739ba..000000000 --- a/app/services/tile_export_service_old.rb +++ /dev/null @@ -1,556 +0,0 @@ -module TileExportServiceOld - BBOX_PADDING = 0.1 - KEY_QUEUE_STOPS = 'queue_stops' - KEY_QUEUE_SCHEDULES = 'queue_schedules' - KEY_STOPID_GRAPHID = 'stopid_graphid' - IMPORT_LEVEL = 4 - GRAPH_LEVEL = 2 - STOP_PAIRS_TILE_LIMIT = 500_000 - - # kTransitEgress = 4, // Transit egress - # kTransitStation = 5, // Transit station - # kMultiUseTransitPlatform = 6, // Multi-use transit platform (rail and bus) - NODE_TYPES = { - StopEgress: 4, - Stop: 5, - StopPlatform: 6 - } - - VT = Valhalla::Mjolnir::Transit::VehicleType - VEHICLE_TYPES = { - tram: VT::Tram, - tram_service: VT::Tram, - metro: VT::Metro, - rail: VT::Rail, - suburban_railway: VT::Rail, - bus: VT::Bus, - trolleybys_service: VT::Bus, - express_bus_service: VT::Bus, - local_bus_service: VT::Bus, - bus_service: VT::Bus, - shuttle_bus: VT::Bus, - demand_and_response_bus_service: VT::Bus, - regional_bus_service: VT::Bus, - ferry: VT::Ferry, - cablecar: VT::CableCar, - gondola: VT::Gondola, - funicular: VT::Funicular - } - - class TileValueError < StandardError - end - - class OriginEqualsDestinationError < TileValueError - end - - class MissingGraphIDError < TileValueError - end - - class MissingRouteError < TileValueError - end - - class MissingShapeError < TileValueError - end - - class MissingTripError < TileValueError - end - - class InvalidTimeError < TileValueError - end - - class TileBuilder - attr_accessor :tile - def initialize(tilepath, tile, feed_version_ids: nil) - @tilepath = tilepath - @tile = tile - # filters - @feed_version_ids = feed_version_ids || [] - # globally unique indexes - @stopid_graphid ||= {} - @graphid_stopid ||= {} - @trip_index ||= TileUtils::DigestIndex.new(bits: 24) - @block_index ||= TileUtils::DigestIndex.new(start: 1, bits: 20) - # tile unique indexes - @route_index = {} - @shape_index = {} - end - - def log(msg) - puts "tile #{@tile}: #{msg}" - end - - def debug(msg) - puts "tile #{@tile} debug: #{msg}" - end - - def build_stops - # TODO: - # max graph_ids in a tile - t = Time.now - - # New tile - tileset = TileUtils::TileSet.new(@tilepath) - tile = tileset.new_tile(GRAPH_LEVEL, @tile) - - Stop - .where_imported_from_feed_version(@feed_version_ids) - .where(parent_stop: nil) - .geometry_within_bbox(bbox_padded(tile.bbox)) - .includes(:stop_platforms, :stop_egresses) - .find_each do |stop| - - # Check if stop is inside tile - stop_tile = TileUtils::GraphID.new(level: GRAPH_LEVEL, lon: stop.coordinates[0], lat: stop.coordinates[1]).tile - if stop_tile != @tile - # debug("skipping stop #{stop.id}: coordinates #{stop.coordinates.join(',')} map to tile #{stop_tile} outside of tile #{@tile}") - next - end - - # Station references - prev_type_graphid = nil - - # Egresses - stop_egresses = stop.stop_egresses.to_a - stop_egresses << StopEgress.new(stop.attributes) if stop_egresses.empty? # generated egress - stop_egresses.each do |stop_egress| - node = make_node(stop_egress) - node.graphid = TileUtils::GraphID.new(level: GRAPH_LEVEL, tile: @tile, index: tile.message.nodes.size).value - node.prev_type_graphid = prev_type_graphid if prev_type_graphid - prev_type_graphid ||= node.graphid - tile.message.nodes << node - end - - # Station - node = make_node(stop) - node.graphid = TileUtils::GraphID.new(level: GRAPH_LEVEL, tile: @tile, index: tile.message.nodes.size).value - node.prev_type_graphid = prev_type_graphid if prev_type_graphid - prev_type_graphid = node.graphid - tile.message.nodes << node - - # Platforms - stop_platforms = stop.stop_platforms.to_a - (stop_platforms << StopPlatform.new(stop.attributes)) if stop_platforms.empty? # station ssps - stop_platforms.each do |stop_platform| - node = make_node(stop_platform) - node.graphid = TileUtils::GraphID.new(level: GRAPH_LEVEL, tile: @tile, index: tile.message.nodes.size).value - node.prev_type_graphid = prev_type_graphid if prev_type_graphid # station_id graphid - @stopid_graphid[stop_platform.id] = node.graphid - @graphid_stopid[node.graphid] = stop_platform.id - tile.message.nodes << node - end - end - - # Write tile - nodes_size = tile.message.nodes.size - tileset.write_tile(tile) if nodes_size > 0 - t = Time.now - t - log("nodes: #{nodes_size} time: #{t.round(2)} (#{(nodes_size/t).to_i} nodes/s)") - return nodes_size - end - - def build_schedules - # Get stop_ids - t = Time.now - tileset = TileUtils::TileSet.new(@tilepath) - base_tile = tileset.read_tile(GRAPH_LEVEL, @tile) - stop_ids = base_tile.message.nodes.map { |node| @graphid_stopid[node.graphid] }.compact - - # Build stop_pairs for each stop_id - tile_ext = 0 - stop_pairs_tile = base_tile # tileset.new_tile(GRAPH_LEVEL, @tile) - stop_pairs_total = 0 - errors = Hash.new(0) - stop_ids.each do |stop_id| - stop_pairs_stop_id_count = 0 - ScheduleStopPair - .where(origin_id: stop_id) - .includes(:origin, :destination, :operator) - .find_in_batches do |ssps| - # Evaluate by active feed version - faster than join or where in (?) - ssps = ssps.select { |ssp| @feed_version_ids.include?(ssp.feed_version_id) } - - # Get unvisited routes for this batch - route_ids = ssps.map(&:route_id).select { |route_id| !@route_index.key?(route_id) } - ssps.each { |ssp| @route_index[ssp.route_id] ||= nil } # set as visited - Route.where(id: route_ids).find_each do |route| # get unvisited - @route_index[route.id] = base_tile.message.routes.size - debug("route: #{route.id} -> #{@route_index[route.id]}") - base_tile.message.routes << make_route(route) - end - - # Get unseen rsps for this batch - rsp_ids = ssps.map(&:route_stop_pattern_id).select { |rsp_id| !@shape_index.key?(rsp_id) } - ssps.each { |ssp| @shape_index[ssp.route_stop_pattern_id] ||= nil } # set as visited - RouteStopPattern.where(id: rsp_ids).find_each do |rsp| - shape = make_shape(rsp) - shape.shape_id = base_tile.message.shapes.size + 1 - @shape_index[rsp.id] = shape.shape_id - debug("shape: #{rsp.id} -> #{@shape_index[rsp.id]}") - base_tile.message.shapes << shape - end - - # Process each ssp - ssps.each do |ssp| - # process ssp and count errors - begin - stop_pairs_tile.message.stop_pairs << make_stop_pair(ssp) - stop_pairs_stop_id_count += 1 - stop_pairs_total += 1 - rescue TileValueError => e - errors[e.class.name.to_sym] += 1 - log("error: ssp #{ssp.id}: #{e}") - rescue StandardError => e - errors[e.class.name.to_sym] += 1 - log("error: ssp #{ssp.id}: #{e}") - end - end - - # Write supplement tile, start new tile - if stop_pairs_tile.message.stop_pairs.size > STOP_PAIRS_TILE_LIMIT - if stop_pairs_tile != base_tile - debug("writing tile ext #{tile_ext}: #{stop_pairs_tile.message.stop_pairs.size} stop_pairs") - tileset.write_tile(stop_pairs_tile, ext: tile_ext) - tile_ext += 1 - end - stop_pairs_tile = tileset.new_tile(GRAPH_LEVEL, @tile) - end - end - # Done for this stop - debug("stop_pairs for stop_id #{stop_id}: #{stop_pairs_stop_id_count}") - end - - # Write dangling supplement tile - if stop_pairs_tile != base_tile && stop_pairs_tile.message.stop_pairs.size > 0 - debug("writing tile ext #{tile_ext}: #{stop_pairs_tile.message.stop_pairs.size} stop_pairs") - tileset.write_tile(stop_pairs_tile, ext: tile_ext) - end - - # Write the base tile - debug("writing tile base: #{base_tile.message.nodes.size} nodes, #{base_tile.message.routes.size} routes, #{base_tile.message.shapes.size} shapes, #{base_tile.message.stop_pairs.size} stop_pairs (#{stop_pairs_total} tile total)") - tileset.write_tile(base_tile) - - # Write tile - t = Time.now - t - error_txt = ([errors.values.sum.to_s] + errors.map { |k,v| "#{k}: #{v}" }).join(' ') - log("#{base_tile.message.nodes.size} nodes, #{base_tile.message.routes.size} routes, #{base_tile.message.shapes.size} shapes, #{stop_pairs_total} stop_pairs, errors #{error_txt}, time: #{t.round(2)} (#{(stop_pairs_total/t).to_i} stop_pairs/s)") - return stop_pairs_total - end - - private - - def seconds_since_midnight(value) - h,m,s = value.split(':').map(&:to_i) - h * 3600 + m * 60 + s - end - - def color_to_int(value) - match = /(\h{6})/.match(value.to_s) - match ? match[0].to_i(16) : nil - end - - # bbox padding - def bbox_padded(bbox) - ymin, xmin, ymax, xmax = bbox - padding = BBOX_PADDING - [ymin-padding, xmin-padding, ymax+padding, xmax+padding] - end - - # make entity methods - def make_stop_pair(ssp) - # TODO: - # skip if origin_departure_time < frequency_start_time - # skip if bad time information - # add < and > to onestop_ids - destination_graphid = @stopid_graphid[ssp.destination_id] - origin_graphid = @stopid_graphid[ssp.origin_id] - fail OriginEqualsDestinationError.new("origin_graphid #{origin_graphid} == destination_graphid #{destination_graphid}") if origin_graphid == destination_graphid - fail MissingGraphIDError.new("missing origin_graphid for stop #{ssp.origin_id}") unless origin_graphid - fail MissingGraphIDError.new("missing destination_graphid for stop #{ssp.destination_id}") unless destination_graphid - - route_index = @route_index[ssp.route_id] - fail MissingRouteError.new("missing route_index for route #{ssp.route_id}") unless route_index - - shape_id = @shape_index[ssp.route_stop_pattern_id] - fail MissingShapeError.new("missing shape for rsp #{ssp.route_stop_pattern_id}") unless shape_id - - trip_id = @trip_index.check(ssp.trip) - fail MissingTripError.new("missing trip_id for trip #{ssp.trip}") unless trip_id - - destination_arrival_time = seconds_since_midnight(ssp.destination_arrival_time) - origin_departure_time = seconds_since_midnight(ssp.origin_departure_time) - fail InvalidTimeError.new("origin_departure_time #{origin_departure_time} > destination_arrival_time #{destination_arrival_time}") if origin_departure_time > destination_arrival_time - - block_id = @block_index.check(ssp.block_id) - - # Make SSP - params = {} - # bool bikes_allowed = 1; - # uint32 block_id = 2; - # params[:block_id] = block_id - # uint32 destination_arrival_time = 3; - params[:destination_arrival_time] = destination_arrival_time - # uint64 destination_graphid = 4; - params[:destination_graphid] = destination_graphid - # string destination_onestop_id = 5; - params[:destination_onestop_id] = ssp.destination.onestop_id - # string operated_by_onestop_id = 6; - params[:operated_by_onestop_id] = ssp.operator.onestop_id - # uint32 origin_departure_time = 7; - params[:origin_departure_time] = origin_departure_time - # uint64 origin_graphid = 8; - params[:origin_graphid] = origin_graphid - # string origin_onestop_id = 9; - params[:origin_onestop_id] = ssp.origin.onestop_id - # uint32 route_index = 10; - params[:route_index] = route_index - # repeated uint32 service_added_dates = 11; - params[:service_added_dates] = ssp.service_added_dates.map(&:jd) - # repeated bool service_days_of_week = 12; - params[:service_days_of_week] = ssp.service_days_of_week - # uint32 service_end_date = 13; - params[:service_end_date] = ssp.service_end_date.jd - # repeated uint32 service_except_dates = 14; - params[:service_except_dates] = ssp.service_except_dates.map(&:jd) - # uint32 service_start_date = 15; - params[:service_start_date] = ssp.service_start_date.jd - # string trip_headsign = 16; - params[:trip_headsign] = ssp.trip_headsign - # uint32 trip_id = 17; - params[:trip_id] = trip_id - # bool wheelchair_accessible = 18; - params[:wheelchair_accessible] = true # !!(ssp.wheelchair_accessible) - # uint32 shape_id = 20; - params[:shape_id] = shape_id - # float origin_dist_traveled = 21; - params[:origin_dist_traveled] = ssp.origin_dist_traveled if ssp.origin_dist_traveled - # float destination_dist_traveled = 22; - params[:destination_dist_traveled] = ssp.destination_dist_traveled if ssp.destination_dist_traveled - if ssp.frequency_headway_seconds - # protobuf doesn't define frequency_start_time - # uint32 frequency_end_time = 23; - params[:frequency_end_time] = seconds_since_midnight(ssp.frequency_end_time) - # uint32 frequency_headway_seconds = 24; - params[:frequency_headway_seconds] = ssp.frequency_headway_seconds - end - Valhalla::Mjolnir::Transit::StopPair.new(params) - end - - def make_shape(rsp) - params = {} - # uint32 shape_id = 1; - # bytes encoded_shape = 2; - # reverse coordinates - reversed = rsp.geometry[:coordinates].map { |a,b| [b,a] } - params[:encoded_shape] = TileUtils::Shape7.encode(reversed) - Valhalla::Mjolnir::Transit::Shape.new(params) - end - - def make_route(route) - # TODO: - # skip if unknown vehicle_type - params = {} - # string name = 1; - params[:name] = route.name - # string onestop_id = 2; - params[:onestop_id] = route.onestop_id - # string operated_by_name = 3; - params[:operated_by_name] = route.operator.name - # string operated_by_onestop_id = 4; - params[:operated_by_onestop_id] = route.operator.onestop_id - # string operated_by_website = 5; - params[:operated_by_website] = route.operator.website - # uint32 route_color = 6; - params[:route_color] = color_to_int(route.color || 'FFFFFF') - # string route_desc = 7; - params[:route_desc] = route.tags["route_desc"] - # string route_long_name = 8; - params[:route_long_name] = route.tags["route_long_name"] || route.name - # uint32 route_text_color = 9; - params[:route_text_color] = color_to_int(route.tags["route_text_color"]) - # VehicleType vehicle_type = 10; - params[:vehicle_type] = VEHICLE_TYPES[route.vehicle_type.to_sym] || VT::Bus - Valhalla::Mjolnir::Transit::Route.new(params.compact) - end - - def make_node(stop) - params = {} - # float lon = 1; - params[:lon] = stop.coordinates[0] - # float lat = 2; - params[:lat] = stop.coordinates[1] - # uint32 type = 3; - params[:type] = NODE_TYPES[stop.class.name.to_sym] - # uint64 graphid = 4; - # set in build_stops - # uint64 prev_type_graphid = 5; - # set in build_stops - # string name = 6; - params[:name] = stop.name - # string onestop_id = 7; - params[:onestop_id] = stop.onestop_id - # uint64 osm_way_id = 8; - params[:osm_way_id] = stop.osm_way_id - # string timezone = 9; - params[:timezone] = stop.timezone - # bool wheelchair_boarding = 10; - params[:wheelchair_boarding] = true - # bool generated = 11; - if stop.instance_of?(StopEgress) && !stop.persisted? - params[:onestop_id] = "#{stop.onestop_id}>" - params[:generated] = true - end - if stop.instance_of?(StopPlatform) && !stop.persisted? - params[:onestop_id] = "#{stop.onestop_id}<" - # params[:generated] = true # not set for platforms - end - # uint32 traversability = 12; - if stop.instance_of?(StopEgress) - params[:traversability] = 3 - end - Valhalla::Mjolnir::Transit::Node.new(params.compact) - end - end - - def self.tile_build_stops(tilepath, feed_version_ids: nil) - redis = Redis.new - while tile = redis.rpop(KEY_QUEUE_STOPS) - tile = tile.to_i - builder = TileBuilder.new(tilepath, tile, feed_version_ids: feed_version_ids) - nodes_size = builder.build_stops - if nodes_size > 0 - redis.rpush(KEY_QUEUE_SCHEDULES, tile) - stopid_graphid = builder.instance_variable_get('@stopid_graphid') - stopid_graphid.each_slice(1000) { |i| redis.hmset(KEY_STOPID_GRAPHID, i.flatten) } - end - remaining = redis.llen(KEY_QUEUE_STOPS) - puts "remaining: ~#{remaining}" - end - end - - def self.tile_build_schedules(tilepath, feed_version_ids: nil) - # stopid_graphid - stopid_graphid = {} - graphid_stopid = {} - redis = Redis.new - cursor = nil - while cursor != '0' - cursor, data = redis.hscan(KEY_STOPID_GRAPHID, cursor, count: 1_000) - data.each do |k,v| - k = k.to_i - v = v.to_i - stopid_graphid[k] = v - graphid_stopid[v] = k - end - end - # queue - while tile = redis.rpop(KEY_QUEUE_SCHEDULES) - tile = tile.to_i - builder = TileBuilder.new(tilepath, tile, feed_version_ids: feed_version_ids) - builder.instance_variable_set('@stopid_graphid', stopid_graphid) - builder.instance_variable_set('@graphid_stopid', graphid_stopid) - builder.build_schedules - remaining = redis.llen(KEY_QUEUE_SCHEDULES) - puts "remaining: ~#{remaining}" - end - end - - def self.export_tiles(tilepath, thread_count: nil, feeds: nil, feed_versions: nil, tiles: nil) - # Debug - # ActiveRecord::Base.logger = Logger.new(STDOUT) - # ActiveRecord::Base.logger.level = Logger::DEBUG - - # Avoid autoload issues in threads - Stop.connection - StopPlatform.connection - StopEgress.connection - Route.connection - Operator.connection - RouteStopPattern.connection - EntityImportedFromFeed.connection - ScheduleStopPair.connection - - # Filter by feed/feed_version - feed_version_ids = [] - if feed_versions - feed_version_ids = feed_versions.map(&:id) - elsif feeds - feed_version_ids = feeds.map(&:active_feed_version_id) - else - feed_version_ids = Feed.where_active_feed_version_import_level(IMPORT_LEVEL).pluck(:active_feed_version_id) - end - - # Build bboxes - puts "Selecting tiles..." - count_stops = Set.new - stop_platforms = Hash.new { |h,k| h[k] = Set.new } - stop_egresses = Hash.new { |h,k| h[k] = Set.new } - tiles = Set.new(tiles) - if tiles.empty? - count = 1 - total = feed_version_ids.size - FeedVersion.where(id: feed_version_ids).includes(:feed).find_each do |feed_version| - feed = feed_version.feed - fvtiles = Set.new - Stop.where_imported_from_feed_version(feed_version).find_each do |stop| - if stop.is_a?(StopPlatform) - stop_platforms[stop.parent_stop_id] << stop.id - elsif stop.is_a?(StopEgress) - stop_egresses[stop.parent_stop_id] << stop.id - else - count_stops << stop.id - fvtiles << TileUtils::GraphID.new(level: GRAPH_LEVEL, lon: stop.coordinates[0], lat: stop.coordinates[1]).tile - end - end - puts "\t(#{count}/#{total}) #{feed_version.feed.onestop_id} #{feed_version.sha1}: #{fvtiles.size} tiles" - tiles += fvtiles - count += 1 - end - end - - # TODO: Filter stop_platforms/stop_egresses by feed_version - count_egresses = count_stops.map { |i| stop_egresses[i].empty? ? 1 : stop_egresses[i].size }.sum - count_platforms = count_stops.map { |i| stop_platforms[i].empty? ? 1 : stop_platforms[i].size }.sum - puts "Tiles to build: #{tiles.size}" - puts "Expected:" - puts "\tstops: #{count_stops.size}" - puts "\tplatforms: #{stop_platforms.map { |k,v| v.size }.sum}" - puts "\tegresses: #{stop_egresses.map { |k,v| v.size }.sum}" - puts "\tnodes: #{count_stops.size + count_egresses + count_platforms}" - puts "\tstopid-graphid: #{count_platforms}" - - # Clear - count_stops.clear - stop_platforms.clear - stop_egresses.clear - # stopid_graphid = Hash[redis.hgetall('stopid_graphid').map { |k,v| [k.to_i, v.to_i] }] - # expected_stops = Set.new - # count_stops.each { |i| expected_stops += (stop_platforms[i].empty? ? [i].to_set : stop_platforms[i]) } - # missing = stopid_graphid.keys.to_set - expected_stops - - # Setup queue - thread_count ||= 1 - redis = Redis.new - redis.del(KEY_QUEUE_STOPS) - redis.del(KEY_QUEUE_SCHEDULES) - redis.del(KEY_STOPID_GRAPHID) - tiles.each_slice(1000) { |i| redis.rpush(KEY_QUEUE_STOPS, i) } - - # Build stops for each tile. - puts "\n===== Stops =====\n" - workers = (0...thread_count).map do - fork { tile_build_stops(tilepath, feed_version_ids: feed_version_ids) } - end - workers.each { |pid| Process.wait(pid) } - - puts "\nStops finished. Schedule tile queue: #{redis.llen(KEY_QUEUE_SCHEDULES)} stopid-graphid mappings: #{redis.hlen(KEY_STOPID_GRAPHID)}" - - # Build schedule, routes, shapes for each tile. - puts "\n===== Routes, Shapes, StopPairs =====\n" - workers = (0...thread_count).map do - fork { tile_build_schedules(tilepath, feed_version_ids: feed_version_ids) } - end - workers.each { |pid| Process.wait(pid) } - - puts "Done!" - end -end \ No newline at end of file