diff --git a/README.md b/README.md index 05ba66a..e0a20f2 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Known simply as "Bogie", this project is a system for tracking and analyzing tra ## Sub Projects -### CSV MUM +### CSVMUM Marshal and unmarshal CSV files to and from Go structs, using reflection, tags, and custom parsers [README](./pkg/csvmum/README.md) diff --git a/pkg/gtfs/README.md b/pkg/gtfs/README.md index 4f6c016..993deb6 100644 --- a/pkg/gtfs/README.md +++ b/pkg/gtfs/README.md @@ -1,4 +1,4 @@ # GTFS Parser Built for [Bogie](../../README.md) -Working branch [gtfs](https://github.com/bridgelightcloud/bogie/blob/gtfs/pkg/gtfs/README.md) \ No newline at end of file +[GTFS Reference](https://gtfs.org/documentation/schedule/reference/) \ No newline at end of file diff --git a/pkg/gtfs/agency.go b/pkg/gtfs/agency.go new file mode 100644 index 0000000..770b5d8 --- /dev/null +++ b/pkg/gtfs/agency.go @@ -0,0 +1,36 @@ +package gtfs + +import ( + "fmt" +) + +type Agency struct { + ID string `json:"agencyId,omitempty" csv:"agency_id"` + Name string `json:"agencyName" csv:"agency_name"` + URL string `json:"agencyUrl" csv:"agency_url"` + Timezone string `json:"agencyTimezone" csv:"agency_timezone"` + Lang string `json:"agencyLang,omitempty" csv:"agency_lang"` + Phone string `json:"agencyPhone,omitempty" csv:"agency_phone"` + FareURL string `json:"agencyFareUrl,omitempty" csv:"agency_fare_url"` + AgencyEmail string `json:"agencyEmail,omitempty" csv:"agency_email"` +} + +func (a Agency) key() string { + return a.ID +} + +func (a Agency) validate() errorList { + var errs errorList + + if a.Name == "" { + errs.add(fmt.Errorf("agency name is required")) + } + if a.URL == "" { + errs.add(fmt.Errorf("agency URL is required")) + } + if a.Timezone == "" { + errs.add(fmt.Errorf("agency timezone is required")) + } + + return errs +} diff --git a/pkg/gtfs/calendar.go b/pkg/gtfs/calendar.go new file mode 100644 index 0000000..29238d5 --- /dev/null +++ b/pkg/gtfs/calendar.go @@ -0,0 +1,23 @@ +package gtfs + +type Calendar struct { + ServiceID string `json:"serviceId" csv:"service_id"` + Monday int `json:"monday" csv:"monday"` + Tuesday int `json:"tuesday" csv:"tuesday"` + Wednesday int `json:"wednesday" csv:"wednesday"` + Thursday int `json:"thursday" csv:"thursday"` + Friday int `json:"friday" csv:"friday"` + Saturday int `json:"saturday" csv:"saturday"` + Sunday int `json:"sunday" csv:"sunday"` + StartDate Date `json:"startDate" csv:"start_date"` + EndDate Date `json:"endDate" csv:"end_date"` +} + +func (c Calendar) key() string { + return c.ServiceID +} + +func (c Calendar) validate() errorList { + var errs errorList + return errs +} diff --git a/pkg/gtfs/calendardate.go b/pkg/gtfs/calendardate.go new file mode 100644 index 0000000..1a6982d --- /dev/null +++ b/pkg/gtfs/calendardate.go @@ -0,0 +1,29 @@ +package gtfs + +import "fmt" + +type CalendarDate struct { + ServiceID string `json:"serviceId" csv:"service_id"` + Date Date `json:"date" csv:"date"` + ExceptionType int `json:"exceptionType" csv:"exception_type"` +} + +func (c CalendarDate) key() string { + return c.ServiceID +} + +func (c CalendarDate) validate() errorList { + var errs errorList + + if c.ServiceID == "" { + errs.add(fmt.Errorf("service ID is required")) + } + if c.Date.IsZero() { + errs.add(fmt.Errorf("date is required")) + } + if c.ExceptionType != 1 && c.ExceptionType != 2 { + errs.add(fmt.Errorf("invalid exception type: %d", c.ExceptionType)) + } + + return errs +} diff --git a/pkg/gtfs/collection.go b/pkg/gtfs/collection.go new file mode 100644 index 0000000..65b50d4 --- /dev/null +++ b/pkg/gtfs/collection.go @@ -0,0 +1,42 @@ +package gtfs + +import ( + "fmt" + + "github.com/google/uuid" +) + +func Overview(c map[string]GTFSSchedule) string { + var o string + + for sid, s := range c { + o += fmt.Sprintf("Schedule %s\n", sid[0:4]) + o += fmt.Sprintf(" %d agencies\n", len(s.Agencies)) + o += fmt.Sprintf(" %d stops\n", len(s.Stops)) + o += fmt.Sprintf(" %d routes\n", len(s.Routes)) + o += fmt.Sprintf(" %d calendar entries\n", len(s.Calendar)) + o += fmt.Sprintf(" %d calendar dates\n", len(s.CalendarDates)) + o += fmt.Sprintf(" %d trips\n", len(s.Trips)) + o += fmt.Sprintf(" %d stop times\n", len(s.StopTimes)) + o += fmt.Sprintf(" %d levels\n", len(s.Levels)) + o += fmt.Sprintf(" %d errors\n", len(s.errors)) + o += "\n" + } + + return o +} + +func CreateGTFSCollection(zipFiles []string) (map[string]GTFSSchedule, error) { + sc := make(map[string]GTFSSchedule) + + for _, path := range zipFiles { + s, err := OpenScheduleFromZipFile(path) + if err != nil { + return sc, err + } + + sc[uuid.NewString()] = s + } + + return sc, nil +} diff --git a/pkg/gtfs/level.go b/pkg/gtfs/level.go new file mode 100644 index 0000000..b50dd3f --- /dev/null +++ b/pkg/gtfs/level.go @@ -0,0 +1,29 @@ +package gtfs + +import ( + "fmt" + "math" +) + +type Level struct { + ID string `json:"levelId" csv:"level_id"` + Index float64 `json:"levelIndex" csv:"level_index"` + Name string `json:"levelName,omitempty" csv:"level_name"` +} + +func (l Level) key() string { + return l.ID +} + +func (l Level) validate() errorList { + var errs errorList + + if l.ID == "" { + errs.add(fmt.Errorf("missing level_id")) + } + if l.Index == math.Inf(-1) { + errs.add(fmt.Errorf("invalid index valie")) + } + + return errs +} diff --git a/pkg/gtfs/record.go b/pkg/gtfs/record.go new file mode 100644 index 0000000..ea24655 --- /dev/null +++ b/pkg/gtfs/record.go @@ -0,0 +1,49 @@ +package gtfs + +import ( + "fmt" + "io" + + "github.com/bridgelightcloud/bogie/pkg/csvmum" +) + +type record interface { + key() string + validate() errorList +} + +func parse[T record](f io.Reader, records map[string]T, errors *errorList) { + csvm, err := csvmum.NewUnmarshaler[T](f) + if err != nil { + errors.add(fmt.Errorf("error creating unmarshaler for file: %w", err)) + return + } + + for { + var r T + + err = csvm.Unmarshal(&r) + if err == io.EOF { + break + } + if err != nil { + errors.add(fmt.Errorf("error unmarshalling file: %w", err)) + continue + } + + errs := r.validate() + if errs != nil { + for _, e := range errs { + errors.add(fmt.Errorf("invalid record: %w", e)) + } + continue + } + + if _, ok := records[r.key()]; ok { + errors.add(fmt.Errorf("duplicate key: %s", r.key())) + continue + } + + records[r.key()] = r + } +} diff --git a/pkg/gtfs/route.go b/pkg/gtfs/route.go new file mode 100644 index 0000000..7e03e46 --- /dev/null +++ b/pkg/gtfs/route.go @@ -0,0 +1,44 @@ +package gtfs + +import ( + "fmt" +) + +type Route struct { + ID string `json:"routeId" csv:"route_id"` + AgencyID string `json:"agencyId" csv:"agency_id"` + ShortName string `json:"routeShortName" csv:"route_short_name"` + LongName string `json:"routeLongName" csv:"route_long_name"` + Desc string `json:"routeDesc,omitempty" csv:"route_desc"` + Type string `json:"routeType" csv:"route_type"` + URL string `json:"routeUrl,omitempty" csv:"route_url"` + Color string `json:"routeColor,omitempty" csv:"route_color"` + TextColor string `json:"routeTextColor,omitempty" csv:"route_text_color"` + SortOrder string `json:"routeSortOrder,omitempty" csv:"route_sort_order"` + ContinuousPickup string `json:"continuousPickup,omitempty" csv:"continuous_pickup"` + ContinuousDropOff string `json:"continuousDropOff,omitempty" csv:"continuous_drop_off"` + NetworkID string `json:"networkId,omitempty" csv:"network_id"` +} + +func (r Route) key() string { + return r.ID +} + +func (r Route) validate() errorList { + var errs errorList + + if r.ID == "" { + errs.add(fmt.Errorf("route ID is required")) + } + if r.ShortName == "" { + errs.add(fmt.Errorf("route short name is required")) + } + if r.LongName == "" { + errs.add(fmt.Errorf("route long name is required")) + } + if r.Type == "" { + errs.add(fmt.Errorf("route type is required")) + } + + return errs +} diff --git a/pkg/gtfs/schedule.go b/pkg/gtfs/schedule.go new file mode 100644 index 0000000..dd2c621 --- /dev/null +++ b/pkg/gtfs/schedule.go @@ -0,0 +1,88 @@ +package gtfs + +import ( + "archive/zip" + "fmt" +) + +type GTFSSchedule struct { + // Required files + Agencies map[string]Agency + Stops map[string]Stop + Routes map[string]Route + Calendar map[string]Calendar + CalendarDates map[string]CalendarDate + Trips map[string]Trip + StopTimes map[string]StopTime + Levels map[string]Level + + unusedFiles []string + errors errorList + warnings errorList +} + +func (s GTFSSchedule) Errors() errorList { + return s.errors +} + +type gtfsSpec[R record] struct { + set func(*GTFSSchedule, map[string]R) +} + +type fileParser interface { + parseFile(*zip.File, *GTFSSchedule, *errorList) +} + +func (spec gtfsSpec[R]) parseFile(f *zip.File, schedule *GTFSSchedule, errors *errorList) { + r, err := f.Open() + if err != nil { + errors.add(fmt.Errorf("error opening file: %w", err)) + return + } + defer r.Close() + + records := make(map[string]R) + + parse(r, records, errors) + + spec.set(schedule, records) +} + +var gtfsSpecs = map[string]fileParser{ + "agency.txt": gtfsSpec[Agency]{set: func(s *GTFSSchedule, r map[string]Agency) { s.Agencies = r }}, + "stops.txt": gtfsSpec[Stop]{set: func(s *GTFSSchedule, r map[string]Stop) { s.Stops = r }}, + "routes.txt": gtfsSpec[Route]{set: func(s *GTFSSchedule, r map[string]Route) { s.Routes = r }}, + "calendar.txt": gtfsSpec[Calendar]{set: func(s *GTFSSchedule, r map[string]Calendar) { s.Calendar = r }}, + "calendar_dates.txt": gtfsSpec[CalendarDate]{set: func(s *GTFSSchedule, r map[string]CalendarDate) { s.CalendarDates = r }}, + "trips.txt": gtfsSpec[Trip]{set: func(s *GTFSSchedule, r map[string]Trip) { s.Trips = r }}, + "stop_times.txt": gtfsSpec[StopTime]{set: func(s *GTFSSchedule, r map[string]StopTime) { s.StopTimes = r }}, + "levels.txt": gtfsSpec[Level]{set: func(s *GTFSSchedule, r map[string]Level) { s.Levels = r }}, +} + +func OpenScheduleFromZipFile(fn string) (GTFSSchedule, error) { + r, err := zip.OpenReader(fn) + if err != nil { + return GTFSSchedule{}, err + } + defer r.Close() + + s := parseSchedule(r) + + return s, nil +} + +func parseSchedule(r *zip.ReadCloser) GTFSSchedule { + var s GTFSSchedule + + for _, f := range r.File { + spec := gtfsSpecs[f.Name] + if spec == nil { + s.unusedFiles = append(s.unusedFiles, f.Name) + s.warnings.add(fmt.Errorf("unused file: %s", f.Name)) + continue + } + spec.parseFile(f, &s, &s.errors) + } + + return s +} diff --git a/pkg/gtfs/stop.go b/pkg/gtfs/stop.go new file mode 100644 index 0000000..e8f1ee2 --- /dev/null +++ b/pkg/gtfs/stop.go @@ -0,0 +1,50 @@ +package gtfs + +import ( + "fmt" +) + +type Stop struct { + ID string `json:"stopId" csv:"stop_id"` + Code string `json:"stopCode,omitempty" csv:"stop_code"` + Name string `json:"stopName" csv:"stop_name"` + TTSName string `json:"TTSStopName,omitempty" csv:"tts_stop_name"` + Desc string `json:"stopDesc,omitempty" csv:"stop_desc"` + Latitude string `json:"latitude" csv:"stop_lat"` + Longitude string `json:"longitude" csv:"stop_lon"` + ZoneID string `json:"zoneId,omitempty" csv:"zone_id"` + URL string `json:"stopUrl,omitempty" csv:"stop_url"` + LocationType int `json:"locationType,omitempty" csv:"location_type"` + ParentStation string `json:"parentStation" csv:"parent_station"` + Timezone string `json:"stopTimezone,omitempty" csv:"stop_timezone"` + WheelchairBoarding string `json:"wheelchairBoarding,omitempty" csv:"wheelchair_boarding"` + LevelID string `json:"levelId,omitempty" csv:"level_id"` + PlatformCode string `json:"platformCode,omitempty" csv:"platform_code"` +} + +func (s Stop) key() string { + return s.ID +} + +func (s Stop) validate() errorList { + var errs errorList + + if s.ID == "" { + errs.add(fmt.Errorf("stop ID is required")) + } + if s.Name == "" { + if s.LocationType == StopPlatform || s.LocationType == Station || s.LocationType == EntranceExit { + errs.add(fmt.Errorf("stop name is required for location type %d", s.LocationType)) + } + } + // if !s.Coords.IsValid() { + // if s.LocationType == StopPlatform || s.LocationType == Station || s.LocationType == EntranceExit { + // errs.add(fmt.Errorf("invalid stop coordinates for location type %d", s.LocationType)) + // } + // } + if s.LocationType < StopPlatform || s.LocationType > BoardingArea { + errs.add(fmt.Errorf("invalid location type: %d", s.LocationType)) + } + + return errs +} diff --git a/pkg/gtfs/stoptime.go b/pkg/gtfs/stoptime.go new file mode 100644 index 0000000..b267a3d --- /dev/null +++ b/pkg/gtfs/stoptime.go @@ -0,0 +1,46 @@ +package gtfs + +import ( + "fmt" +) + +type StopTime struct { + TripID string `json:"tripId" csv:"trip_id"` + ArrivalTime Time `json:"arrivalTime,omitempty" csv:"arrival_time"` + DepartureTime Time `json:"departureTime,omitempty" csv:"departure_time"` + StopID string `json:"stopId" csv:"stop_id"` + LocationGroupID string `json:"locationGroupId" csv:"location_group_id"` + LocationID string `json:"locationId" csv:"location_id"` + StopSequence int `json:"stopSequence" csv:"stop_sequence"` + StopHeadsign string `json:"stopHeadsign" csv:"stop_headsign"` + StartPickupDropOffWindow Time `json:"startPickupDropOffWindow" csv:"start_pickup_drop_off_window"` + EndPickupDropOffWindow Time `json:"endPickupDropOffWindow" csv:"end_pickup_drop_off_window"` + PickupType *int `json:"pickupType" csv:"pickup_type"` + DropOffType *int `json:"dropOffType" csv:"drop_off_type"` + ContinuousPickup *int `json:"continuousPickup" csv:"continuous_pickup"` + ContinuousDropOff *int `json:"continuousDropOff" csv:"continuous_drop_off"` + ShapeDistTraveled *float64 `json:"shapeDistTraveled" csv:"shape_dist_traveled"` + Timepoint *int `json:"timepoint" csv:"timepoint"` + PickupBookingRuleId string `json:"pickupBookingRuleId" csv:"pickup_booking_rule_id"` + DropOffBookingRuleId string `json:"dropOffBookingRuleId" csv:"drop_off_booking_rule_id"` +} + +func (st StopTime) key() string { + return fmt.Sprintf("%s-%d", st.TripID, st.StopSequence) +} + +func (st StopTime) validate() errorList { + var errs errorList + + if st.TripID == "" { + errs.add(fmt.Errorf("trip ID is required")) + } + if st.StopSequence < 0 { + errs.add(fmt.Errorf("stop sequence must be greater than or equal to 0")) + } + if st.StopID == "" { + errs.add(fmt.Errorf("stop ID is required")) + } + + return errs +} diff --git a/pkg/gtfs/trip.go b/pkg/gtfs/trip.go new file mode 100644 index 0000000..c20cf7f --- /dev/null +++ b/pkg/gtfs/trip.go @@ -0,0 +1,38 @@ +package gtfs + +import ( + "fmt" +) + +type Trip struct { + RouteID string `json:"routeId,omitempty" csv:"route_id,omitempty"` + ServiceID string `json:"serviceId,omitempty" csv:"service_id,omitempty"` + ID string `json:"tripId" csv:"trip_id"` + Headsign string `json:"tripHeadsign" csv:"trip_headsign"` + ShortName string `json:"tripShortName" csv:"trip_short_name"` + DirectionID int `json:"directionId" csv:"direction_id"` + BlockID string `json:"blockId" csv:"block_id"` + ShapeID string `json:"shapeId" csv:"shape_id"` + WheelchairAccessible int `json:"wheelchairAccessible" csv:"wheelchair_accessible"` + BikesAllowed int `json:"bikesAllowed" csv:"bikes_allowed"` +} + +func (t Trip) key() string { + return t.ID +} + +func (t Trip) validate() errorList { + var errs errorList + + if t.ID == "" { + errs.add(fmt.Errorf("trip ID is required")) + } + if t.ServiceID == "" { + errs.add(fmt.Errorf("trip service id is required")) + } + if t.ID == "" { + errs.add(fmt.Errorf("trip ID is required")) + } + + return errs +} diff --git a/pkg/gtfs/types.go b/pkg/gtfs/types.go new file mode 100644 index 0000000..db88155 --- /dev/null +++ b/pkg/gtfs/types.go @@ -0,0 +1,354 @@ +package gtfs + +import ( + "fmt" + "regexp" + "strconv" + "strings" + "time" +) + +var validColor = regexp.MustCompile(`(?i)^[a-f\d]{6}$`) + +func ParseColor(v string, c *string) error { + f := strings.TrimSpace(v) + if !validColor.MatchString(f) { + return fmt.Errorf("invalid color: %s", v) + } + + *c = strings.ToUpper(f) + return nil +} + +var validCurrencyCodes = map[string]int{ + "AED": 2, + "AFN": 2, + "ALL": 2, + "AMD": 2, + "ANG": 2, + "AOA": 2, + "ARS": 2, + "AUD": 2, + "AWG": 2, + "AZN": 2, + "BAM": 2, + "BBD": 2, + "BDT": 2, + "BGN": 2, + "BHD": 3, + "BIF": 0, + "BMD": 2, + "BND": 2, + "BOB": 2, + "BOV": 2, + "BRL": 2, + "BSD": 2, + "BTN": 2, + "BWP": 2, + "BYN": 2, + "BZD": 2, + "CAD": 2, + "CDF": 2, + "CHE": 2, + "CHF": 2, + "CHW": 2, + "CLF": 4, + "CLP": 0, + "CNY": 2, + "COP": 2, + "COU": 2, + "CRC": 2, + "CUP": 2, + "CVE": 2, + "CZK": 2, + "DJF": 0, + "DKK": 2, + "DOP": 2, + "DZD": 2, + "EGP": 2, + "ERN": 2, + "ETB": 2, + "EUR": 2, + "FJD": 2, + "FKP": 2, + "GBP": 2, + "GEL": 2, + "GHS": 2, + "GIP": 2, + "GMD": 2, + "GNF": 0, + "GTQ": 2, + "GYD": 2, + "HKD": 2, + "HNL": 2, + "HTG": 2, + "HUF": 2, + "IDR": 2, + "ILS": 2, + "INR": 2, + "IQD": 3, + "IRR": 2, + "ISK": 0, + "JMD": 2, + "JOD": 3, + "JPY": 0, + "KES": 2, + "KGS": 2, + "KHR": 2, + "KMF": 0, + "KPW": 2, + "KRW": 0, + "KWD": 3, + "KYD": 2, + "KZT": 2, + "LAK": 2, + "LBP": 2, + "LKR": 2, + "LRD": 2, + "LSL": 2, + "LYD": 3, + "MAD": 2, + "MDL": 2, + "MGA": 2, + "MKD": 2, + "MMK": 2, + "MNT": 2, + "MOP": 2, + "MRU": 2, + "MUR": 2, + "MVR": 2, + "MWK": 2, + "MXN": 2, + "MXV": 2, + "MYR": 2, + "MZN": 2, + "NAD": 2, + "NGN": 2, + "NIO": 2, + "NOK": 2, + "NPR": 2, + "NZD": 2, + "OMR": 3, + "PAB": 2, + "PEN": 2, + "PGK": 2, + "PHP": 2, + "PKR": 2, + "PLN": 2, + "PYG": 0, + "QAR": 2, + "RON": 2, + "RSD": 2, + "RUB": 2, + "RWF": 0, + "SAR": 2, + "SBD": 2, + "SCR": 2, + "SDG": 2, + "SEK": 2, + "SGD": 2, + "SHP": 2, + "SLE": 2, + "SOS": 2, + "SRD": 2, + "SSP": 2, + "STN": 2, + "SVC": 2, + "SYP": 2, + "SZL": 2, + "THB": 2, + "TJS": 2, + "TMT": 2, + "TND": 3, + "TOP": 2, + "TRY": 2, + "TTD": 2, + "TWD": 2, + "TZS": 2, + "UAH": 2, + "UGX": 0, + "USD": 2, + "USN": 2, + "UYI": 0, + "UYU": 2, + "UYW": 4, + "UZS": 2, + "VED": 2, + "VES": 2, + "VND": 0, + "VUV": 0, + "WST": 2, + "YER": 2, + "ZAR": 2, + "ZMW": 2, + "ZWG": 2, +} + +func ParseCurrencyCode(v string, c *string) error { + f := strings.TrimSpace(v) + f = strings.ToUpper(f) + if _, ok := validCurrencyCodes[f]; !ok { + return fmt.Errorf("invalid currency code: %s", v) + } + + *c = f + return nil +} + +type Date struct { + time.Time +} + +var dateFormat = "20060102" + +func (d Date) MarshalText() ([]byte, error) { + return []byte(d.Format(dateFormat)), nil +} + +func (d *Date) UnmarshalText(text []byte) error { + p, err := time.Parse(dateFormat, string(text)) + if err != nil { + return fmt.Errorf("invalid date value: %s", text) + } + d.Time = p + return nil +} + +func (d Date) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf("%d", d.Unix())), nil +} + +func (d *Date) UnmarshalJSON(data []byte) error { + i, err := strconv.ParseInt(string(data), 10, 64) + if err != nil { + return fmt.Errorf("invalid date value: %s", string(data)) + } + *d = Date{time.Unix(i, 0)} + return nil +} + +type Time struct { + time.Time +} + +var timeFormat = "15:04:05" + +func (t Time) MarshalText() ([]byte, error) { + timeStr := t.Format(timeFormat) + + if d := t.Time.Day(); d > 1 { + hrs := strconv.Itoa(t.Hour() + 24) + return []byte(hrs + timeStr[2:]), nil + } + + return []byte(timeStr), nil +} + +func (t *Time) UnmarshalText(text []byte) error { + timeStr := string(text) + + p, err := time.Parse(timeFormat, timeStr) + + if err != nil { + if len(timeStr) < 8 { + return fmt.Errorf("invalid time value: %s", text) + } + hrs := timeStr[:2] + h, err := strconv.Atoi(hrs) + if err != nil || h < 24 { + return fmt.Errorf("invalid time value: %s", text) + } + + timeStr = strconv.Itoa(h-24) + timeStr[2:] + + p, err = time.Parse(timeFormat, timeStr) + + if err != nil { + return fmt.Errorf("invalid time value: %s", text) + } + + t.Time = p.AddDate(0, 0, 1) + } else { + t.Time = p + } + + return nil +} + +func (t Time) MarshalJSON() ([]byte, error) { + if t.IsZero() { + return []byte("null"), nil + } + return []byte(fmt.Sprintf("%d", t.Unix())), nil +} + +func (t *Time) UnmarshalJSON(data []byte) error { + if str := string(data); str == "null" { + t.Time = time.Time{} + } else if i, err := strconv.ParseInt(str, 10, 64); err == nil { + *t = Time{Time: time.Unix(i, 0)} + } else { + return fmt.Errorf("invalid time value: %s", str) + } + + return nil +} + +type enumBounds struct { + L int + U int +} + +var ( + Availability enumBounds = enumBounds{0, 1} + Available int = 0 + Unavailable int = 1 + + BikesAllowed enumBounds = enumBounds{0, 2} + NoInfo int = 0 + AtLeastOneBicycleAccomodated int = 1 + NoBicyclesAllowed int = 2 + + ContinuousPickup enumBounds = enumBounds{0, 3} + ContinuousDropOff enumBounds = enumBounds{0, 3} + DropOffType enumBounds = enumBounds{0, 3} + PickupType enumBounds = enumBounds{0, 3} + RegularlyScheduled int = 0 + NoneAvailable int = 1 + MustPhoneAgency int = 2 + MustCoordinate int = 3 + + DirectionID enumBounds = enumBounds{0, 1} + OneDirection int = 0 + OppositeDirection int = 1 + + ExceptionType enumBounds = enumBounds{1, 2} + Added int = 1 + Removed int = 2 + + Timepoint enumBounds = enumBounds{0, 1} + ApproximateTime int = 0 + ExactTime int = 1 + + LocationType enumBounds = enumBounds{0, 4} + StopPlatform int = 0 + Station int = 1 + EntranceExit int = 2 + GenericNode int = 3 + BoardingArea int = 4 + + WheelchairAccessible enumBounds = enumBounds{0, 2} + UnknownAccessibility int = 0 + AtLeastOneWheelchairAccomodated int = 1 + NoWheelchairsAccomodated int = 2 +) + +type errorList []error + +func (e *errorList) add(err error) error { + if err == nil { + return err + } + *e = append(*e, err) + return err +} diff --git a/pkg/gtfs/types_test.go b/pkg/gtfs/types_test.go new file mode 100644 index 0000000..3be78e9 --- /dev/null +++ b/pkg/gtfs/types_test.go @@ -0,0 +1,526 @@ +package gtfs + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestParseColor(t *testing.T) { + t.Parallel() + + tt := []struct { + value string + expectedErr error + expectedColor string + }{{ + value: "000000", + expectedErr: nil, + expectedColor: "000000", + }, { + value: "FFFFFF", + expectedErr: nil, + expectedColor: "FFFFFF", + }, { + value: "123456", + expectedErr: nil, + expectedColor: "123456", + }, { + value: "ABCDEF", + expectedErr: nil, + expectedColor: "ABCDEF", + }, { + value: "abc123", + expectedErr: nil, + expectedColor: "ABC123", + }, { + value: "abC14D", + expectedErr: nil, + expectedColor: "ABC14D", + }, { + value: "1234567", + expectedErr: fmt.Errorf("invalid color: 1234567"), + expectedColor: "", + }, { + value: "ABCDEF1", + expectedErr: fmt.Errorf("invalid color: ABCDEF1"), + expectedColor: "", + }, { + value: "12345", + expectedErr: fmt.Errorf("invalid color: 12345"), + expectedColor: "", + }, { + value: "ABCDE", + expectedErr: fmt.Errorf("invalid color: ABCDE"), + expectedColor: "", + }, { + value: "12345G", + expectedErr: fmt.Errorf("invalid color: 12345G"), + expectedColor: "", + }, { + value: "ABCDEG", + expectedErr: fmt.Errorf("invalid color: ABCDEG"), + expectedColor: "", + }, { + value: "", + expectedErr: fmt.Errorf("invalid color: "), + expectedColor: "", + }, { + value: " 04FE2B", + expectedErr: nil, + expectedColor: "04FE2B", + }, { + value: "#A5FF32", + expectedErr: fmt.Errorf("invalid color: #A5FF32"), + expectedColor: "", + }} + + for _, tc := range tt { + tc := tc + + t.Run(t.Name(), func(t *testing.T) { + t.Parallel() + + assert := assert.New(t) + + var c string + err := ParseColor(tc.value, &c) + + assert.Equal(tc.expectedErr, err) + assert.Equal(tc.expectedColor, c) + }) + } +} + +func TestParseCurrencyCode(t *testing.T) { + t.Parallel() + + tt := []struct { + value string + expectedErr error + expectedCode string + }{{ + value: "USD", + expectedErr: nil, + expectedCode: "USD", + }, { + value: "usd", + expectedErr: nil, + expectedCode: "USD", + }, { + value: "uSd", + expectedErr: nil, + expectedCode: "USD", + }, { + value: "usd ", + expectedErr: nil, + expectedCode: "USD", + }, { + value: "USD1", + expectedErr: fmt.Errorf("invalid currency code: %s", "USD1"), + expectedCode: "", + }, { + value: " ", + expectedErr: fmt.Errorf("invalid currency code: %s", " "), + expectedCode: "", + }, { + value: "", + expectedErr: fmt.Errorf("invalid currency code: %s", ""), + expectedCode: "", + }} + + for _, tc := range tt { + tc := tc + + t.Run(t.Name(), func(t *testing.T) { + t.Parallel() + + assert := assert.New(t) + + var c string + err := ParseCurrencyCode(tc.value, &c) + + assert.Equal(tc.expectedErr, err) + assert.Equal(tc.expectedCode, c) + }) + } +} + +func TestDateMarshalText(t *testing.T) { + t.Parallel() + + tt := []struct { + name string + date Date + out []byte + err error + }{{ + name: "valid date", + date: Date{Time: time.Date(2004, 11, 27, 0, 0, 0, 0, time.UTC)}, + out: []byte("20041127"), + err: nil, + }, { + name: "zero date", + date: Date{Time: time.Time{}}, + out: []byte("00010101"), + err: nil, + }} + + for _, tc := range tt { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + assert := assert.New(t) + + mt, err := tc.date.MarshalText() + assert.Equal(tc.out, mt) + assert.Equal(tc.err, err) + + }) + } +} + +func TestDateUnmarshalText(t *testing.T) { + t.Parallel() + + tt := []struct { + name string + in []byte + date Date + err error + }{{ + name: "valid date", + in: []byte("20241127"), + date: Date{Time: time.Date(2024, 11, 27, 0, 0, 0, 0, time.UTC)}, + err: nil, + }, { + name: "zero date?", + in: []byte("00010101"), + date: Date{Time: time.Time{}}, + err: nil, + }, { + name: "invalid date", + in: []byte("Nov 27, 2024"), + date: Date{Time: time.Time{}}, + err: fmt.Errorf("invalid date value: Nov 27, 2024"), + }} + + for _, tc := range tt { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + assert := assert.New(t) + + var d Date + err := d.UnmarshalText(tc.in) + + assert.Equal(tc.date, d) + assert.Equal(tc.err, err) + }) + } +} + +func TestDateMarshalJSON(t *testing.T) { + t.Parallel() + + tt := []struct { + name string + date Date + out []byte + err error + }{{ + name: "valid date", + date: Date{Time: time.Date(2024, 11, 27, 0, 0, 0, 0, time.UTC)}, + out: []byte("1732665600"), + err: nil, + }, { + name: "zero date", + date: Date{Time: time.Time{}}, + out: []byte("-62135596800"), + err: nil, + }} + + for _, tc := range tt { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + assert := assert.New(t) + + dm, err := tc.date.MarshalJSON() + + assert.Equal(tc.out, dm) + assert.Equal(tc.err, err) + }) + } +} + +func TestDateUnmarshalJSON(t *testing.T) { + t.Parallel() + + tt := []struct { + name string + in []byte + date Date + err error + }{{ + name: "valid date", + in: []byte("1732665600"), + date: Date{Time: time.Date(2024, 11, 27, 0, 0, 0, 0, time.Local)}, + err: nil, + }, { + name: "zero date", + in: []byte("-62135596800"), + date: Date{Time: time.Date(1, 1, 1, 0, 0, 0, 0, time.Local)}, + err: nil, + }, { + name: "invalid date", + in: []byte("x"), + // date: Date{Time: time.Date(1, 1, 1, 0, 0, 0, 0, time.Local)}, + date: Date{Time: time.Time{}}, + err: fmt.Errorf("invalid date value: x"), + }} + + for _, tc := range tt { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + assert := assert.New(t) + + var d Date + err := d.UnmarshalJSON(tc.in) + + assert.Equal(tc.date, d) + assert.Equal(tc.err, err) + }) + } +} + +func TestTimeMarshalText(t *testing.T) { + t.Parallel() + + tt := []struct { + name string + time Time + out []byte + err error + }{{ + name: "time under 24 hrs", + time: Time{Time: time.Date(1, 1, 1, 12, 55, 30, 0, time.UTC)}, + out: []byte("12:55:30"), + err: nil, + }, { + name: "time over 24 hrs", + time: Time{Time: time.Date(1, 1, 2, 1, 34, 22, 0, time.UTC)}, + out: []byte("25:34:22"), + err: nil, + }, { + name: "zero time", + time: Time{Time: time.Time{}}, + out: []byte("00:00:00"), + err: nil, + }} + + for _, tc := range tt { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + assert := assert.New(t) + + mt, err := tc.time.MarshalText() + assert.Equal(tc.out, mt) + assert.Equal(tc.err, err) + + }) + } +} + +func TestTimeUnmarshalText(t *testing.T) { + t.Parallel() + + tt := []struct { + name string + in []byte + time Time + err error + }{{ + name: "time under 24 hrs", + in: []byte("17:23:22"), + time: Time{Time: time.Date(0, 1, 1, 17, 23, 22, 0, time.UTC)}, + err: nil, + }, { + name: "time over 24 hrs", + in: []byte("25:34:22"), + time: Time{Time: time.Date(0, 1, 2, 1, 34, 22, 0, time.UTC)}, + }, { + name: "zero time", + in: []byte("00:00:00"), + time: Time{Time: time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC)}, + // time: Time{Time: time.Time{}}, + err: nil, + }, { + name: "invalid time", + in: []byte("09:34 AM"), + time: Time{Time: time.Time{}}, + err: fmt.Errorf("invalid time value: 09:34 AM"), + }, { + name: "invalid time over 24 hrs", + in: []byte("24:77:22"), + time: Time{Time: time.Time{}}, + err: fmt.Errorf("invalid time value: 24:77:22"), + }, { + name: "invalid time under 48 hrs", + in: []byte("48:34:22"), + time: Time{Time: time.Time{}}, + err: fmt.Errorf("invalid time value: 48:34:22"), + }} + + for _, tc := range tt { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + assert := assert.New(t) + + var time Time + err := time.UnmarshalText(tc.in) + + assert.Equal(tc.time, time) + assert.Equal(tc.err, err) + }) + } +} + +func TestTimeMarshalJSON(t *testing.T) { + t.Parallel() + + tt := []struct { + name string + time Time + out []byte + err error + }{{ + name: "valid time", + time: Time{Time: time.Date(1, 1, 1, 12, 57, 44, 0, time.UTC)}, + out: []byte("-62135550136"), + err: nil, + }, { + name: "zero time", + time: Time{Time: time.Time{}}, + out: []byte("null"), + err: nil, + }} + + for _, tc := range tt { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + assert := assert.New(t) + + dm, err := tc.time.MarshalJSON() + + assert.Equal(tc.out, dm) + assert.Equal(tc.err, err) + }) + } +} + +func TestTimeUnmarshalJSON(t *testing.T) { + t.Parallel() + + tt := []struct { + name string + in []byte + time Time + err error + }{{ + name: "valid time", + in: []byte("-62135550136"), + time: Time{Time: time.Date(1, 1, 1, 12, 57, 44, 0, time.Local)}, err: nil, + }, { + name: "zero time", + in: []byte("null"), + time: Time{Time: time.Time{}}, + err: nil, + }, { + name: "invalid time", + in: []byte("x"), + time: Time{Time: time.Time{}}, + err: fmt.Errorf("invalid time value: x"), + }} + + for _, tc := range tt { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + assert := assert.New(t) + + var d Time + err := d.UnmarshalJSON(tc.in) + + assert.Equal(tc.time, d) + assert.Equal(tc.err, err) + }) + } +} + +func TestErrorList(t *testing.T) { + t.Parallel() + + tt := []struct { + errList errorList + err error + expList errorList + }{{ + errList: errorList{}, + err: nil, + expList: errorList{}, + }, { + errList: errorList{fmt.Errorf("error 1")}, + err: nil, + expList: errorList{fmt.Errorf("error 1")}, + }, { + errList: errorList{}, + err: fmt.Errorf("error 1"), + expList: errorList{fmt.Errorf("error 1")}, + }, { + errList: errorList{fmt.Errorf("error 1")}, + err: fmt.Errorf("error 2"), + expList: errorList{fmt.Errorf("error 1"), fmt.Errorf("error 2")}, + }, { + errList: errorList{fmt.Errorf("error 1"), fmt.Errorf("error 2")}, + err: fmt.Errorf("error 3"), + expList: errorList{fmt.Errorf("error 1"), fmt.Errorf("error 2"), fmt.Errorf("error 3")}, + }} + + for _, tc := range tt { + tc := tc + + t.Run(t.Name(), func(t *testing.T) { + t.Parallel() + + assert := assert.New(t) + + err := tc.errList.add(tc.err) + + assert.Equal(tc.err, err) + assert.Equal(tc.expList, tc.errList) + }) + } +} diff --git a/pkg/util/print.go b/pkg/util/print.go new file mode 100644 index 0000000..b687bc9 --- /dev/null +++ b/pkg/util/print.go @@ -0,0 +1,12 @@ +package util + +import ( + "encoding/json" + "fmt" +) + +func PrintAsFormattedJSON(data any) { + if j, err := json.MarshalIndent(data, "", " "); err == nil { + fmt.Println(string(j)) + } +} diff --git a/pkg/util/time.go b/pkg/util/time.go new file mode 100644 index 0000000..c23af77 --- /dev/null +++ b/pkg/util/time.go @@ -0,0 +1,16 @@ +package util + +import ( + "fmt" + "time" +) + +type TimeTracker func() + +func TrackTime(message string) TimeTracker { + st := time.Now() + return func() { + et := time.Now() + fmt.Printf("Time taken to %s: %s\n", message, et.Sub(st).String()) + } +} diff --git a/tools/gtfs/main.go b/tools/gtfs/main.go new file mode 100644 index 0000000..5b693bd --- /dev/null +++ b/tools/gtfs/main.go @@ -0,0 +1,47 @@ +package main + +import ( + "fmt" + "log" + "os" + "path/filepath" + + "github.com/bridgelightcloud/bogie/pkg/gtfs" + "github.com/bridgelightcloud/bogie/pkg/util" +) + +func main() { + tt := util.TrackTime("create GTFS collection") + defer tt() + + gtfsDir := "gtfs_files" + + if _, err := os.Stat(gtfsDir); err != nil { + log.Fatalf("Error finding %s: %s \n", gtfsDir, err.Error()) + } + + zipFiles, err := filepath.Glob(filepath.Join(gtfsDir, "*.zip")) + if err != nil { + log.Fatalf("Malformed file path: %s\n", err.Error()) + } + + col, err := gtfs.CreateGTFSCollection(zipFiles) + if err != nil { + log.Fatalf("Error creating GTFS schedule collection: %s\n", err) + tt() + } + + fmt.Println(gtfs.Overview(col)) + + errFile, err := os.Create("gtfs_files/gtfs_errors.txt") + if err != nil { + log.Fatalf("Error creating error file: %s\n", err.Error()) + } + defer errFile.Close() + + for _, e := range col { + for _, err := range e.Errors() { + errFile.WriteString(fmt.Sprintf("%s\n", err)) + } + } +} diff --git a/tools/uuid/uuidgen.go b/tools/uuid/uuidgen.go index 3806b4e..d04696c 100644 --- a/tools/uuid/uuidgen.go +++ b/tools/uuid/uuidgen.go @@ -24,7 +24,7 @@ func main() { for i := 0; i < c; i++ { uuid := uuid.New() fmt.Println(uuid.String()) - var arr [16]byte = uuid - fmt.Printf("%#v\n\n", arr) + // var arr [16]byte = uuid + // fmt.Printf("%#v\n\n", arr) } }