From 13a0838632d7ec241c75f193b656ed7c6b8e4152 Mon Sep 17 00:00:00 2001 From: Ryan Choi Date: Mon, 14 Sep 2015 13:27:30 -0700 Subject: [PATCH] update schema for geo --- schema/schema_gnip.json | 1879 ++++++++++++++++++++++++++++++++++++++- utils.py | 24 +- 2 files changed, 1900 insertions(+), 3 deletions(-) diff --git a/schema/schema_gnip.json b/schema/schema_gnip.json index c023631..0b794c3 100644 --- a/schema/schema_gnip.json +++ b/schema/schema_gnip.json @@ -1 +1,1878 @@ -[{"type": "string", "name": "body", "mode": "nullable"}, {"fields": [{"type": "string", "name": "link", "mode": "nullable"}], "type": "record", "name": "inReplyTo", "mode": "nullable"}, {"type": "timestamp", "name": "postedTime", "mode": "nullable"}, {"type": "integer", "name": "retweetCount", "mode": "nullable"}, {"fields": [{"type": "string", "name": "displayName", "mode": "nullable"}, {"type": "string", "name": "link", "mode": "nullable"}], "type": "record", "name": "generator", "mode": "nullable"}, {"fields": [{"fields": [{"type": "integer", "name": "indices", "mode": "repeated"}, {"type": "string", "name": "screen_name", "mode": "nullable"}, {"type": "integer", "name": "id", "mode": "nullable"}, {"type": "string", "name": "name", "mode": "nullable"}, {"type": "string", "name": "id_str", "mode": "nullable"}], "type": "record", "name": "user_mentions", "mode": "repeated"}, {"fields": [{"type": "string", "name": "source_status_id_str", "mode": "nullable"}, {"type": "string", "name": "expanded_url", "mode": "nullable"}, {"type": "string", "name": "display_url", "mode": "nullable"}, {"type": "string", "name": "url", "mode": "nullable"}, {"type": "string", "name": "media_url_https", "mode": "nullable"}, {"type": "integer", "name": "source_status_id", "mode": "nullable"}, {"fields": [{"type": "integer", "name": "aspect_ratio", "mode": "repeated"}, {"type": "integer", "name": "duration_millis", "mode": "nullable"}, {"fields": [{"type": "string", "name": "url", "mode": "nullable"}, {"type": "integer", "name": "bitrate", "mode": "nullable"}, {"type": "string", "name": "content_type", "mode": "nullable"}], "type": "record", "name": "variants", "mode": "repeated"}], "type": "record", "name": "video_info", "mode": "nullable"}, {"type": "string", "name": "id_str", "mode": "nullable"}, {"fields": [{"fields": [{"type": "integer", "name": "h", "mode": "nullable"}, {"type": "integer", "name": "w", "mode": "nullable"}, {"type": "string", "name": "resize", "mode": "nullable"}], "type": "record", "name": "small", "mode": "nullable"}, {"fields": [{"type": "integer", "name": "h", "mode": "nullable"}, {"type": "integer", "name": "w", "mode": "nullable"}, {"type": "string", "name": "resize", "mode": "nullable"}], "type": "record", "name": "large", "mode": "nullable"}, {"fields": [{"type": "integer", "name": "h", "mode": "nullable"}, {"type": "integer", "name": "w", "mode": "nullable"}, {"type": "string", "name": "resize", "mode": "nullable"}], "type": "record", "name": "medium", "mode": "nullable"}, {"fields": [{"type": "integer", "name": "h", "mode": "nullable"}, {"type": "integer", "name": "w", "mode": "nullable"}, {"type": "string", "name": "resize", "mode": "nullable"}], "type": "record", "name": "thumb", "mode": "nullable"}], "type": "record", "name": "sizes", "mode": "nullable"}, {"type": "integer", "name": "indices", "mode": "repeated"}, {"type": "string", "name": "type", "mode": "nullable"}, {"type": "integer", "name": "id", "mode": "nullable"}, {"type": "string", "name": "media_url", "mode": "nullable"}], "type": "record", "name": "media", "mode": "repeated"}, {"fields": [{"type": "integer", "name": "indices", "mode": "repeated"}, {"type": "string", "name": "text", "mode": "nullable"}], "type": "record", "name": "hashtags", "mode": "repeated"}, {"fields": [{"type": "integer", "name": "indices", "mode": "repeated"}, {"type": "string", "name": "text", "mode": "nullable"}], "type": "record", "name": "symbols", "mode": "repeated"}, {"fields": [{"type": "integer", "name": "woeid", "mode": "nullable"}, {"type": "string", "name": "name", "mode": "nullable"}, {"type": "string", "name": "countryCode", "mode": "nullable"}, {"type": "string", "name": "url", "mode": "nullable"}, {"type": "string", "name": "country", "mode": "nullable"}, {"fields": [{"type": "integer", "name": "code", "mode": "nullable"}, {"type": "string", "name": "name", "mode": "nullable"}], "type": "record", "name": "placeType", "mode": "nullable"}, {"type": "integer", "name": "parentid", "mode": "nullable"}], "type": "record", "name": "trends", "mode": "repeated"}, {"fields": [{"type": "string", "name": "url", "mode": "nullable"}, {"type": "integer", "name": "indices", "mode": "repeated"}, {"type": "string", "name": "expanded_url", "mode": "nullable"}, {"type": "string", "name": "display_url", "mode": "nullable"}], "type": "record", "name": "urls", "mode": "repeated"}], "type": "record", "name": "twitter_entities", "mode": "nullable"}, {"fields": [{"fields": [{"type": "string", "name": "tag", "mode": "nullable"}, {"type": "string", "name": "value", "mode": "nullable"}], "type": "record", "name": "matching_rules", "mode": "repeated"}, {"fields": [{"type": "string", "name": "url", "mode": "nullable"}, {"type": "integer", "name": "expanded_status", "mode": "nullable"}, {"type": "string", "name": "expanded_url", "mode": "nullable"}], "type": "record", "name": "urls", "mode": "repeated"}, {"fields": [{"type": "string", "name": "value", "mode": "nullable"}], "type": "record", "name": "language", "mode": "nullable"}], "type": "record", "name": "gnip", "mode": "nullable"}, {"type": "integer", "name": "favoritesCount", "mode": "nullable"}, {"fields": [{"type": "string", "name": "body", "mode": "nullable"}, {"fields": [{"type": "string", "name": "link", "mode": "nullable"}], "type": "record", "name": "inReplyTo", "mode": "nullable"}, {"fields": [{"type": "string", "name": "displayName", "mode": "nullable"}, {"type": "string", "name": "link", "mode": "nullable"}], "type": "record", "name": "generator", "mode": "nullable"}, {"fields": [{"fields": [{"type": "integer", "name": "indices", "mode": "repeated"}, {"type": "string", "name": "screen_name", "mode": "nullable"}, {"type": "integer", "name": "id", "mode": "nullable"}, {"type": "string", "name": "name", "mode": "nullable"}, {"type": "string", "name": "id_str", "mode": "nullable"}], "type": "record", "name": "user_mentions", "mode": "repeated"}, {"fields": [{"type": "string", "name": "source_status_id_str", "mode": "nullable"}, {"type": "string", "name": "expanded_url", "mode": "nullable"}, {"type": "string", "name": "display_url", "mode": "nullable"}, {"type": "string", "name": "url", "mode": "nullable"}, {"type": "string", "name": "media_url_https", "mode": "nullable"}, {"type": "integer", "name": "source_status_id", "mode": "nullable"}, {"fields": [{"type": "integer", "name": "aspect_ratio", "mode": "repeated"}, {"type": "integer", "name": "duration_millis", "mode": "nullable"}, {"fields": [{"type": "string", "name": "url", "mode": "nullable"}, {"type": "integer", "name": "bitrate", "mode": "nullable"}, {"type": "string", "name": "content_type", "mode": "nullable"}], "type": "record", "name": "variants", "mode": "repeated"}], "type": "record", "name": "video_info", "mode": "nullable"}, {"type": "string", "name": "id_str", "mode": "nullable"}, {"fields": [{"fields": [{"type": "integer", "name": "h", "mode": "nullable"}, {"type": "integer", "name": "w", "mode": "nullable"}, {"type": "string", "name": "resize", "mode": "nullable"}], "type": "record", "name": "small", "mode": "nullable"}, {"fields": [{"type": "integer", "name": "h", "mode": "nullable"}, {"type": "integer", "name": "w", "mode": "nullable"}, {"type": "string", "name": "resize", "mode": "nullable"}], "type": "record", "name": "large", "mode": "nullable"}, {"fields": [{"type": "integer", "name": "h", "mode": "nullable"}, {"type": "integer", "name": "w", "mode": "nullable"}, {"type": "string", "name": "resize", "mode": "nullable"}], "type": "record", "name": "medium", "mode": "nullable"}, {"fields": [{"type": "integer", "name": "h", "mode": "nullable"}, {"type": "integer", "name": "w", "mode": "nullable"}, {"type": "string", "name": "resize", "mode": "nullable"}], "type": "record", "name": "thumb", "mode": "nullable"}], "type": "record", "name": "sizes", "mode": "nullable"}, {"type": "integer", "name": "indices", "mode": "repeated"}, {"type": "string", "name": "type", "mode": "nullable"}, {"type": "integer", "name": "id", "mode": "nullable"}, {"type": "string", "name": "media_url", "mode": "nullable"}], "type": "record", "name": "media", "mode": "repeated"}, {"fields": [{"type": "integer", "name": "indices", "mode": "repeated"}, {"type": "string", "name": "text", "mode": "nullable"}], "type": "record", "name": "hashtags", "mode": "repeated"}, {"fields": [{"type": "integer", "name": "indices", "mode": "repeated"}, {"type": "string", "name": "text", "mode": "nullable"}], "type": "record", "name": "symbols", "mode": "repeated"}, {"fields": [{"type": "integer", "name": "woeid", "mode": "nullable"}, {"type": "string", "name": "name", "mode": "nullable"}, {"type": "string", "name": "countryCode", "mode": "nullable"}, {"type": "string", "name": "url", "mode": "nullable"}, {"type": "string", "name": "country", "mode": "nullable"}, {"fields": [{"type": "integer", "name": "code", "mode": "nullable"}, {"type": "string", "name": "name", "mode": "nullable"}], "type": "record", "name": "placeType", "mode": "nullable"}, {"type": "integer", "name": "parentid", "mode": "nullable"}], "type": "record", "name": "trends", "mode": "repeated"}, {"fields": [{"type": "string", "name": "url", "mode": "nullable"}, {"type": "integer", "name": "indices", "mode": "repeated"}, {"type": "string", "name": "expanded_url", "mode": "nullable"}, {"type": "string", "name": "display_url", "mode": "nullable"}], "type": "record", "name": "urls", "mode": "repeated"}], "type": "record", "name": "twitter_entities", "mode": "nullable"}, {"type": "timestamp", "name": "postedTime", "mode": "nullable"}, {"type": "integer", "name": "favoritesCount", "mode": "nullable"}, {"fields": [{"type": "string", "name": "link", "mode": "nullable"}, {"type": "string", "name": "objectType", "mode": "nullable"}, {"type": "timestamp", "name": "postedTime", "mode": "nullable"}, {"type": "string", "name": "id", "mode": "nullable"}, {"type": "string", "name": "summary", "mode": "nullable"}], "type": "record", "name": "object", "mode": "nullable"}, {"fields": [{"type": "string", "name": "preferredUsername", "mode": "nullable"}, {"type": "boolean", "name": "verified", "mode": "nullable"}, {"fields": [{"type": "string", "name": "href", "mode": "nullable"}, {"type": "string", "name": "rel", "mode": "nullable"}], "type": "record", "name": "links", "mode": "repeated"}, {"type": "string", "name": "twitterTimeZone", "mode": "nullable"}, {"type": "timestamp", "name": "postedTime", "mode": "nullable"}, {"type": "string", "name": "image", "mode": "nullable"}, {"type": "integer", "name": "statusesCount", "mode": "nullable"}, {"type": "string", "name": "summary", "mode": "nullable"}, {"type": "string", "name": "languages", "mode": "repeated"}, {"type": "string", "name": "utcOffset", "mode": "nullable"}, {"type": "string", "name": "link", "mode": "nullable"}, {"fields": [{"type": "string", "name": "displayName", "mode": "nullable"}, {"type": "string", "name": "objectType", "mode": "nullable"}], "type": "record", "name": "location", "mode": "nullable"}, {"type": "string", "name": "id", "mode": "nullable"}, {"type": "string", "name": "displayName", "mode": "nullable"}, {"type": "integer", "name": "listedCount", "mode": "nullable"}, {"type": "integer", "name": "followersCount", "mode": "nullable"}, {"type": "integer", "name": "favoritesCount", "mode": "nullable"}, {"type": "integer", "name": "friendsCount", "mode": "nullable"}, {"type": "string", "name": "objectType", "mode": "nullable"}], "type": "record", "name": "actor", "mode": "nullable"}, {"type": "string", "name": "summary", "mode": "nullable"}, {"type": "string", "name": "twitter_lang", "mode": "nullable"}, {"type": "string", "name": "twitter_filter_level", "mode": "nullable"}, {"type": "string", "name": "verb", "mode": "nullable"}, {"type": "string", "name": "link", "mode": "nullable"}, {"fields": [{"type": "string", "name": "twitter_country_code", "mode": "nullable"}, {"type": "string", "name": "displayName", "mode": "nullable"}, {"type": "string", "name": "name", "mode": "nullable"}, {"type": "string", "name": "link", "mode": "nullable"}, {"type": "string", "name": "country_code", "mode": "nullable"}, {"fields": [{"type": "string", "name": "type", "mode": "nullable"}], "type": "record", "name": "geo", "mode": "nullable"}, {"type": "string", "name": "twitter_place_type", "mode": "nullable"}, {"type": "string", "name": "objectType", "mode": "nullable"}], "type": "record", "name": "location", "mode": "nullable"}, {"fields": [{"fields": [{"type": "string", "name": "source_status_id_str", "mode": "nullable"}, {"type": "string", "name": "expanded_url", "mode": "nullable"}, {"type": "string", "name": "display_url", "mode": "nullable"}, {"type": "string", "name": "url", "mode": "nullable"}, {"type": "string", "name": "media_url_https", "mode": "nullable"}, {"type": "integer", "name": "source_status_id", "mode": "nullable"}, {"fields": [{"type": "integer", "name": "aspect_ratio", "mode": "repeated"}, {"type": "integer", "name": "duration_millis", "mode": "nullable"}, {"fields": [{"type": "string", "name": "url", "mode": "nullable"}, {"type": "integer", "name": "bitrate", "mode": "nullable"}, {"type": "string", "name": "content_type", "mode": "nullable"}], "type": "record", "name": "variants", "mode": "repeated"}], "type": "record", "name": "video_info", "mode": "nullable"}, {"type": "string", "name": "id_str", "mode": "nullable"}, {"fields": [{"fields": [{"type": "integer", "name": "h", "mode": "nullable"}, {"type": "integer", "name": "w", "mode": "nullable"}, {"type": "string", "name": "resize", "mode": "nullable"}], "type": "record", "name": "small", "mode": "nullable"}, {"fields": [{"type": "integer", "name": "h", "mode": "nullable"}, {"type": "integer", "name": "w", "mode": "nullable"}, {"type": "string", "name": "resize", "mode": "nullable"}], "type": "record", "name": "large", "mode": "nullable"}, {"fields": [{"type": "integer", "name": "h", "mode": "nullable"}, {"type": "integer", "name": "w", "mode": "nullable"}, {"type": "string", "name": "resize", "mode": "nullable"}], "type": "record", "name": "medium", "mode": "nullable"}, {"fields": [{"type": "integer", "name": "h", "mode": "nullable"}, {"type": "integer", "name": "w", "mode": "nullable"}, {"type": "string", "name": "resize", "mode": "nullable"}], "type": "record", "name": "thumb", "mode": "nullable"}], "type": "record", "name": "sizes", "mode": "nullable"}, {"type": "integer", "name": "indices", "mode": "repeated"}, {"type": "string", "name": "type", "mode": "nullable"}, {"type": "integer", "name": "id", "mode": "nullable"}, {"type": "string", "name": "media_url", "mode": "nullable"}], "type": "record", "name": "media", "mode": "repeated"}], "type": "record", "name": "twitter_extended_entities", "mode": "nullable"}, {"fields": [{"type": "string", "name": "displayName", "mode": "nullable"}, {"type": "string", "name": "link", "mode": "nullable"}, {"type": "string", "name": "objectType", "mode": "nullable"}], "type": "record", "name": "provider", "mode": "nullable"}, {"fields": [{"type": "string", "name": "type", "mode": "nullable"}], "type": "record", "name": "geo", "mode": "nullable"}, {"type": "string", "name": "id", "mode": "nullable"}, {"type": "string", "name": "objectType", "mode": "nullable"}], "type": "record", "name": "object", "mode": "nullable"}, {"fields": [{"type": "string", "name": "preferredUsername", "mode": "nullable"}, {"type": "boolean", "name": "verified", "mode": "nullable"}, {"fields": [{"type": "string", "name": "href", "mode": "nullable"}, {"type": "string", "name": "rel", "mode": "nullable"}], "type": "record", "name": "links", "mode": "repeated"}, {"type": "string", "name": "twitterTimeZone", "mode": "nullable"}, {"type": "timestamp", "name": "postedTime", "mode": "nullable"}, {"type": "string", "name": "image", "mode": "nullable"}, {"type": "integer", "name": "statusesCount", "mode": "nullable"}, {"type": "string", "name": "summary", "mode": "nullable"}, {"type": "string", "name": "languages", "mode": "repeated"}, {"type": "string", "name": "utcOffset", "mode": "nullable"}, {"type": "string", "name": "link", "mode": "nullable"}, {"fields": [{"type": "string", "name": "twitter_country_code", "mode": "nullable"}, {"type": "string", "name": "displayName", "mode": "nullable"}, {"type": "string", "name": "name", "mode": "nullable"}, {"type": "string", "name": "link", "mode": "nullable"}, {"type": "string", "name": "country_code", "mode": "nullable"}, {"fields": [{"type": "string", "name": "type", "mode": "nullable"}], "type": "record", "name": "geo", "mode": "nullable"}, {"type": "string", "name": "twitter_place_type", "mode": "nullable"}, {"type": "string", "name": "objectType", "mode": "nullable"}], "type": "record", "name": "location", "mode": "nullable"}, {"type": "string", "name": "id", "mode": "nullable"}, {"type": "string", "name": "displayName", "mode": "nullable"}, {"type": "integer", "name": "listedCount", "mode": "nullable"}, {"type": "integer", "name": "followersCount", "mode": "nullable"}, {"type": "integer", "name": "favoritesCount", "mode": "nullable"}, {"type": "integer", "name": "friendsCount", "mode": "nullable"}, {"type": "string", "name": "objectType", "mode": "nullable"}], "type": "record", "name": "actor", "mode": "nullable"}, {"type": "string", "name": "twitter_lang", "mode": "nullable"}, {"type": "string", "name": "twitter_filter_level", "mode": "nullable"}, {"type": "string", "name": "verb", "mode": "nullable"}, {"type": "string", "name": "link", "mode": "nullable"}, {"fields": [{"type": "string", "name": "twitter_country_code", "mode": "nullable"}, {"type": "string", "name": "displayName", "mode": "nullable"}, {"type": "string", "name": "name", "mode": "nullable"}, {"type": "string", "name": "link", "mode": "nullable"}, {"type": "string", "name": "country_code", "mode": "nullable"}, {"fields": [{"type": "string", "name": "type", "mode": "nullable"}], "type": "record", "name": "geo", "mode": "nullable"}, {"type": "string", "name": "twitter_place_type", "mode": "nullable"}, {"type": "string", "name": "objectType", "mode": "nullable"}], "type": "record", "name": "location", "mode": "nullable"}, {"fields": [{"fields": [{"type": "string", "name": "source_status_id_str", "mode": "nullable"}, {"type": "string", "name": "expanded_url", "mode": "nullable"}, {"type": "string", "name": "display_url", "mode": "nullable"}, {"type": "string", "name": "url", "mode": "nullable"}, {"type": "string", "name": "media_url_https", "mode": "nullable"}, {"type": "integer", "name": "source_status_id", "mode": "nullable"}, {"fields": [{"type": "integer", "name": "aspect_ratio", "mode": "repeated"}, {"type": "integer", "name": "duration_millis", "mode": "nullable"}, {"fields": [{"type": "string", "name": "url", "mode": "nullable"}, {"type": "integer", "name": "bitrate", "mode": "nullable"}, {"type": "string", "name": "content_type", "mode": "nullable"}], "type": "record", "name": "variants", "mode": "repeated"}], "type": "record", "name": "video_info", "mode": "nullable"}, {"type": "string", "name": "id_str", "mode": "nullable"}, {"fields": [{"fields": [{"type": "integer", "name": "h", "mode": "nullable"}, {"type": "integer", "name": "w", "mode": "nullable"}, {"type": "string", "name": "resize", "mode": "nullable"}], "type": "record", "name": "small", "mode": "nullable"}, {"fields": [{"type": "integer", "name": "h", "mode": "nullable"}, {"type": "integer", "name": "w", "mode": "nullable"}, {"type": "string", "name": "resize", "mode": "nullable"}], "type": "record", "name": "large", "mode": "nullable"}, {"fields": [{"type": "integer", "name": "h", "mode": "nullable"}, {"type": "integer", "name": "w", "mode": "nullable"}, {"type": "string", "name": "resize", "mode": "nullable"}], "type": "record", "name": "medium", "mode": "nullable"}, {"fields": [{"type": "integer", "name": "h", "mode": "nullable"}, {"type": "integer", "name": "w", "mode": "nullable"}, {"type": "string", "name": "resize", "mode": "nullable"}], "type": "record", "name": "thumb", "mode": "nullable"}], "type": "record", "name": "sizes", "mode": "nullable"}, {"type": "integer", "name": "indices", "mode": "repeated"}, {"type": "string", "name": "type", "mode": "nullable"}, {"type": "integer", "name": "id", "mode": "nullable"}, {"type": "string", "name": "media_url", "mode": "nullable"}], "type": "record", "name": "media", "mode": "repeated"}], "type": "record", "name": "twitter_extended_entities", "mode": "nullable"}, {"fields": [{"type": "string", "name": "displayName", "mode": "nullable"}, {"type": "string", "name": "link", "mode": "nullable"}, {"type": "string", "name": "objectType", "mode": "nullable"}], "type": "record", "name": "provider", "mode": "nullable"}, {"fields": [{"type": "string", "name": "type", "mode": "nullable"}], "type": "record", "name": "geo", "mode": "nullable"}, {"type": "string", "name": "id", "mode": "nullable"}, {"type": "string", "name": "objectType", "mode": "nullable"}] +[ + { + "type":"string", + "name":"body", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"link", + "mode":"nullable" + } + ], + "type":"record", + "name":"inReplyTo", + "mode":"nullable" + }, + { + "type":"timestamp", + "name":"postedTime", + "mode":"nullable" + }, + { + "type":"integer", + "name":"retweetCount", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"displayName", + "mode":"nullable" + }, + { + "type":"string", + "name":"link", + "mode":"nullable" + } + ], + "type":"record", + "name":"generator", + "mode":"nullable" + }, + { + "fields":[ + { + "fields":[ + { + "type":"integer", + "name":"indices", + "mode":"repeated" + }, + { + "type":"string", + "name":"screen_name", + "mode":"nullable" + }, + { + "type":"integer", + "name":"id", + "mode":"nullable" + }, + { + "type":"string", + "name":"name", + "mode":"nullable" + }, + { + "type":"string", + "name":"id_str", + "mode":"nullable" + } + ], + "type":"record", + "name":"user_mentions", + "mode":"repeated" + }, + { + "fields":[ + { + "type":"string", + "name":"source_status_id_str", + "mode":"nullable" + }, + { + "type":"string", + "name":"expanded_url", + "mode":"nullable" + }, + { + "type":"string", + "name":"display_url", + "mode":"nullable" + }, + { + "type":"string", + "name":"url", + "mode":"nullable" + }, + { + "type":"string", + "name":"media_url_https", + "mode":"nullable" + }, + { + "type":"integer", + "name":"source_status_id", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"integer", + "name":"aspect_ratio", + "mode":"repeated" + }, + { + "type":"integer", + "name":"duration_millis", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"url", + "mode":"nullable" + }, + { + "type":"integer", + "name":"bitrate", + "mode":"nullable" + }, + { + "type":"string", + "name":"content_type", + "mode":"nullable" + } + ], + "type":"record", + "name":"variants", + "mode":"repeated" + } + ], + "type":"record", + "name":"video_info", + "mode":"nullable" + }, + { + "type":"string", + "name":"id_str", + "mode":"nullable" + }, + { + "fields":[ + { + "fields":[ + { + "type":"integer", + "name":"h", + "mode":"nullable" + }, + { + "type":"integer", + "name":"w", + "mode":"nullable" + }, + { + "type":"string", + "name":"resize", + "mode":"nullable" + } + ], + "type":"record", + "name":"small", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"integer", + "name":"h", + "mode":"nullable" + }, + { + "type":"integer", + "name":"w", + "mode":"nullable" + }, + { + "type":"string", + "name":"resize", + "mode":"nullable" + } + ], + "type":"record", + "name":"large", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"integer", + "name":"h", + "mode":"nullable" + }, + { + "type":"integer", + "name":"w", + "mode":"nullable" + }, + { + "type":"string", + "name":"resize", + "mode":"nullable" + } + ], + "type":"record", + "name":"medium", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"integer", + "name":"h", + "mode":"nullable" + }, + { + "type":"integer", + "name":"w", + "mode":"nullable" + }, + { + "type":"string", + "name":"resize", + "mode":"nullable" + } + ], + "type":"record", + "name":"thumb", + "mode":"nullable" + } + ], + "type":"record", + "name":"sizes", + "mode":"nullable" + }, + { + "type":"integer", + "name":"indices", + "mode":"repeated" + }, + { + "type":"string", + "name":"type", + "mode":"nullable" + }, + { + "type":"integer", + "name":"id", + "mode":"nullable" + }, + { + "type":"string", + "name":"media_url", + "mode":"nullable" + } + ], + "type":"record", + "name":"media", + "mode":"repeated" + }, + { + "fields":[ + { + "type":"integer", + "name":"indices", + "mode":"repeated" + }, + { + "type":"string", + "name":"text", + "mode":"nullable" + } + ], + "type":"record", + "name":"hashtags", + "mode":"repeated" + }, + { + "fields":[ + { + "type":"integer", + "name":"indices", + "mode":"repeated" + }, + { + "type":"string", + "name":"text", + "mode":"nullable" + } + ], + "type":"record", + "name":"symbols", + "mode":"repeated" + }, + { + "fields":[ + { + "type":"integer", + "name":"woeid", + "mode":"nullable" + }, + { + "type":"string", + "name":"name", + "mode":"nullable" + }, + { + "type":"string", + "name":"countryCode", + "mode":"nullable" + }, + { + "type":"string", + "name":"url", + "mode":"nullable" + }, + { + "type":"string", + "name":"country", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"integer", + "name":"code", + "mode":"nullable" + }, + { + "type":"string", + "name":"name", + "mode":"nullable" + } + ], + "type":"record", + "name":"placeType", + "mode":"nullable" + }, + { + "type":"integer", + "name":"parentid", + "mode":"nullable" + } + ], + "type":"record", + "name":"trends", + "mode":"repeated" + }, + { + "fields":[ + { + "type":"string", + "name":"url", + "mode":"nullable" + }, + { + "type":"integer", + "name":"indices", + "mode":"repeated" + }, + { + "type":"string", + "name":"expanded_url", + "mode":"nullable" + }, + { + "type":"string", + "name":"display_url", + "mode":"nullable" + } + ], + "type":"record", + "name":"urls", + "mode":"repeated" + } + ], + "type":"record", + "name":"twitter_entities", + "mode":"nullable" + }, + { + "fields":[ + { + "fields":[ + { + "type":"string", + "name":"tag", + "mode":"nullable" + }, + { + "type":"string", + "name":"value", + "mode":"nullable" + } + ], + "type":"record", + "name":"matching_rules", + "mode":"repeated" + }, + { + "fields":[ + { + "type":"string", + "name":"url", + "mode":"nullable" + }, + { + "type":"integer", + "name":"expanded_status", + "mode":"nullable" + }, + { + "type":"string", + "name":"expanded_url", + "mode":"nullable" + } + ], + "type":"record", + "name":"urls", + "mode":"repeated" + }, + { + "fields":[ + { + "type":"string", + "name":"value", + "mode":"nullable" + } + ], + "type":"record", + "name":"language", + "mode":"nullable" + } + ], + "type":"record", + "name":"gnip", + "mode":"nullable" + }, + { + "type":"integer", + "name":"favoritesCount", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"body", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"link", + "mode":"nullable" + } + ], + "type":"record", + "name":"inReplyTo", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"displayName", + "mode":"nullable" + }, + { + "type":"string", + "name":"link", + "mode":"nullable" + } + ], + "type":"record", + "name":"generator", + "mode":"nullable" + }, + { + "fields":[ + { + "fields":[ + { + "type":"integer", + "name":"indices", + "mode":"repeated" + }, + { + "type":"string", + "name":"screen_name", + "mode":"nullable" + }, + { + "type":"integer", + "name":"id", + "mode":"nullable" + }, + { + "type":"string", + "name":"name", + "mode":"nullable" + }, + { + "type":"string", + "name":"id_str", + "mode":"nullable" + } + ], + "type":"record", + "name":"user_mentions", + "mode":"repeated" + }, + { + "fields":[ + { + "type":"string", + "name":"source_status_id_str", + "mode":"nullable" + }, + { + "type":"string", + "name":"expanded_url", + "mode":"nullable" + }, + { + "type":"string", + "name":"display_url", + "mode":"nullable" + }, + { + "type":"string", + "name":"url", + "mode":"nullable" + }, + { + "type":"string", + "name":"media_url_https", + "mode":"nullable" + }, + { + "type":"integer", + "name":"source_status_id", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"integer", + "name":"aspect_ratio", + "mode":"repeated" + }, + { + "type":"integer", + "name":"duration_millis", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"url", + "mode":"nullable" + }, + { + "type":"integer", + "name":"bitrate", + "mode":"nullable" + }, + { + "type":"string", + "name":"content_type", + "mode":"nullable" + } + ], + "type":"record", + "name":"variants", + "mode":"repeated" + } + ], + "type":"record", + "name":"video_info", + "mode":"nullable" + }, + { + "type":"string", + "name":"id_str", + "mode":"nullable" + }, + { + "fields":[ + { + "fields":[ + { + "type":"integer", + "name":"h", + "mode":"nullable" + }, + { + "type":"integer", + "name":"w", + "mode":"nullable" + }, + { + "type":"string", + "name":"resize", + "mode":"nullable" + } + ], + "type":"record", + "name":"small", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"integer", + "name":"h", + "mode":"nullable" + }, + { + "type":"integer", + "name":"w", + "mode":"nullable" + }, + { + "type":"string", + "name":"resize", + "mode":"nullable" + } + ], + "type":"record", + "name":"large", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"integer", + "name":"h", + "mode":"nullable" + }, + { + "type":"integer", + "name":"w", + "mode":"nullable" + }, + { + "type":"string", + "name":"resize", + "mode":"nullable" + } + ], + "type":"record", + "name":"medium", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"integer", + "name":"h", + "mode":"nullable" + }, + { + "type":"integer", + "name":"w", + "mode":"nullable" + }, + { + "type":"string", + "name":"resize", + "mode":"nullable" + } + ], + "type":"record", + "name":"thumb", + "mode":"nullable" + } + ], + "type":"record", + "name":"sizes", + "mode":"nullable" + }, + { + "type":"integer", + "name":"indices", + "mode":"repeated" + }, + { + "type":"string", + "name":"type", + "mode":"nullable" + }, + { + "type":"integer", + "name":"id", + "mode":"nullable" + }, + { + "type":"string", + "name":"media_url", + "mode":"nullable" + } + ], + "type":"record", + "name":"media", + "mode":"repeated" + }, + { + "fields":[ + { + "type":"integer", + "name":"indices", + "mode":"repeated" + }, + { + "type":"string", + "name":"text", + "mode":"nullable" + } + ], + "type":"record", + "name":"hashtags", + "mode":"repeated" + }, + { + "fields":[ + { + "type":"integer", + "name":"indices", + "mode":"repeated" + }, + { + "type":"string", + "name":"text", + "mode":"nullable" + } + ], + "type":"record", + "name":"symbols", + "mode":"repeated" + }, + { + "fields":[ + { + "type":"integer", + "name":"woeid", + "mode":"nullable" + }, + { + "type":"string", + "name":"name", + "mode":"nullable" + }, + { + "type":"string", + "name":"countryCode", + "mode":"nullable" + }, + { + "type":"string", + "name":"url", + "mode":"nullable" + }, + { + "type":"string", + "name":"country", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"integer", + "name":"code", + "mode":"nullable" + }, + { + "type":"string", + "name":"name", + "mode":"nullable" + } + ], + "type":"record", + "name":"placeType", + "mode":"nullable" + }, + { + "type":"integer", + "name":"parentid", + "mode":"nullable" + } + ], + "type":"record", + "name":"trends", + "mode":"repeated" + }, + { + "fields":[ + { + "type":"string", + "name":"url", + "mode":"nullable" + }, + { + "type":"integer", + "name":"indices", + "mode":"repeated" + }, + { + "type":"string", + "name":"expanded_url", + "mode":"nullable" + }, + { + "type":"string", + "name":"display_url", + "mode":"nullable" + } + ], + "type":"record", + "name":"urls", + "mode":"repeated" + } + ], + "type":"record", + "name":"twitter_entities", + "mode":"nullable" + }, + { + "type":"timestamp", + "name":"postedTime", + "mode":"nullable" + }, + { + "type":"integer", + "name":"favoritesCount", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"link", + "mode":"nullable" + }, + { + "type":"string", + "name":"objectType", + "mode":"nullable" + }, + { + "type":"timestamp", + "name":"postedTime", + "mode":"nullable" + }, + { + "type":"string", + "name":"id", + "mode":"nullable" + }, + { + "type":"string", + "name":"summary", + "mode":"nullable" + } + ], + "type":"record", + "name":"object", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"preferredUsername", + "mode":"nullable" + }, + { + "type":"boolean", + "name":"verified", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"href", + "mode":"nullable" + }, + { + "type":"string", + "name":"rel", + "mode":"nullable" + } + ], + "type":"record", + "name":"links", + "mode":"repeated" + }, + { + "type":"string", + "name":"twitterTimeZone", + "mode":"nullable" + }, + { + "type":"timestamp", + "name":"postedTime", + "mode":"nullable" + }, + { + "type":"string", + "name":"image", + "mode":"nullable" + }, + { + "type":"integer", + "name":"statusesCount", + "mode":"nullable" + }, + { + "type":"string", + "name":"summary", + "mode":"nullable" + }, + { + "type":"string", + "name":"languages", + "mode":"repeated" + }, + { + "type":"string", + "name":"utcOffset", + "mode":"nullable" + }, + { + "type":"string", + "name":"link", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"displayName", + "mode":"nullable" + }, + { + "type":"string", + "name":"objectType", + "mode":"nullable" + } + ], + "type":"record", + "name":"location", + "mode":"nullable" + }, + { + "type":"string", + "name":"id", + "mode":"nullable" + }, + { + "type":"string", + "name":"displayName", + "mode":"nullable" + }, + { + "type":"integer", + "name":"listedCount", + "mode":"nullable" + }, + { + "type":"integer", + "name":"followersCount", + "mode":"nullable" + }, + { + "type":"integer", + "name":"favoritesCount", + "mode":"nullable" + }, + { + "type":"integer", + "name":"friendsCount", + "mode":"nullable" + }, + { + "type":"string", + "name":"objectType", + "mode":"nullable" + } + ], + "type":"record", + "name":"actor", + "mode":"nullable" + }, + { + "type":"string", + "name":"summary", + "mode":"nullable" + }, + { + "type":"string", + "name":"twitter_lang", + "mode":"nullable" + }, + { + "type":"string", + "name":"twitter_filter_level", + "mode":"nullable" + }, + { + "type":"string", + "name":"verb", + "mode":"nullable" + }, + { + "type":"string", + "name":"link", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"twitter_country_code", + "mode":"nullable" + }, + { + "type":"string", + "name":"displayName", + "mode":"nullable" + }, + { + "type":"string", + "name":"name", + "mode":"nullable" + }, + { + "type":"string", + "name":"link", + "mode":"nullable" + }, + { + "type":"string", + "name":"country_code", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"type", + "mode":"nullable" + }, + { + "type":"float", + "name":"lat", + "mode":"nullable" + }, + { + "type":"float", + "name":"long", + "mode":"nullable" + } + ], + "type":"record", + "name":"geo", + "mode":"nullable" + }, + { + "type":"string", + "name":"twitter_place_type", + "mode":"nullable" + }, + { + "type":"string", + "name":"objectType", + "mode":"nullable" + } + ], + "type":"record", + "name":"location", + "mode":"nullable" + }, + { + "fields":[ + { + "fields":[ + { + "type":"string", + "name":"source_status_id_str", + "mode":"nullable" + }, + { + "type":"string", + "name":"expanded_url", + "mode":"nullable" + }, + { + "type":"string", + "name":"display_url", + "mode":"nullable" + }, + { + "type":"string", + "name":"url", + "mode":"nullable" + }, + { + "type":"string", + "name":"media_url_https", + "mode":"nullable" + }, + { + "type":"integer", + "name":"source_status_id", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"integer", + "name":"aspect_ratio", + "mode":"repeated" + }, + { + "type":"integer", + "name":"duration_millis", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"url", + "mode":"nullable" + }, + { + "type":"integer", + "name":"bitrate", + "mode":"nullable" + }, + { + "type":"string", + "name":"content_type", + "mode":"nullable" + } + ], + "type":"record", + "name":"variants", + "mode":"repeated" + } + ], + "type":"record", + "name":"video_info", + "mode":"nullable" + }, + { + "type":"string", + "name":"id_str", + "mode":"nullable" + }, + { + "fields":[ + { + "fields":[ + { + "type":"integer", + "name":"h", + "mode":"nullable" + }, + { + "type":"integer", + "name":"w", + "mode":"nullable" + }, + { + "type":"string", + "name":"resize", + "mode":"nullable" + } + ], + "type":"record", + "name":"small", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"integer", + "name":"h", + "mode":"nullable" + }, + { + "type":"integer", + "name":"w", + "mode":"nullable" + }, + { + "type":"string", + "name":"resize", + "mode":"nullable" + } + ], + "type":"record", + "name":"large", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"integer", + "name":"h", + "mode":"nullable" + }, + { + "type":"integer", + "name":"w", + "mode":"nullable" + }, + { + "type":"string", + "name":"resize", + "mode":"nullable" + } + ], + "type":"record", + "name":"medium", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"integer", + "name":"h", + "mode":"nullable" + }, + { + "type":"integer", + "name":"w", + "mode":"nullable" + }, + { + "type":"string", + "name":"resize", + "mode":"nullable" + } + ], + "type":"record", + "name":"thumb", + "mode":"nullable" + } + ], + "type":"record", + "name":"sizes", + "mode":"nullable" + }, + { + "type":"integer", + "name":"indices", + "mode":"repeated" + }, + { + "type":"string", + "name":"type", + "mode":"nullable" + }, + { + "type":"integer", + "name":"id", + "mode":"nullable" + }, + { + "type":"string", + "name":"media_url", + "mode":"nullable" + } + ], + "type":"record", + "name":"media", + "mode":"repeated" + } + ], + "type":"record", + "name":"twitter_extended_entities", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"displayName", + "mode":"nullable" + }, + { + "type":"string", + "name":"link", + "mode":"nullable" + }, + { + "type":"string", + "name":"objectType", + "mode":"nullable" + } + ], + "type":"record", + "name":"provider", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"type", + "mode":"nullable" + }, + { + "type":"float", + "name":"lat", + "mode":"nullable" + }, + { + "type":"float", + "name":"long", + "mode":"nullable" + } + ], + "type":"record", + "name":"geo", + "mode":"nullable" + }, + { + "type":"string", + "name":"id", + "mode":"nullable" + }, + { + "type":"string", + "name":"objectType", + "mode":"nullable" + } + ], + "type":"record", + "name":"object", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"preferredUsername", + "mode":"nullable" + }, + { + "type":"boolean", + "name":"verified", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"href", + "mode":"nullable" + }, + { + "type":"string", + "name":"rel", + "mode":"nullable" + } + ], + "type":"record", + "name":"links", + "mode":"repeated" + }, + { + "type":"string", + "name":"twitterTimeZone", + "mode":"nullable" + }, + { + "type":"timestamp", + "name":"postedTime", + "mode":"nullable" + }, + { + "type":"string", + "name":"image", + "mode":"nullable" + }, + { + "type":"integer", + "name":"statusesCount", + "mode":"nullable" + }, + { + "type":"string", + "name":"summary", + "mode":"nullable" + }, + { + "type":"string", + "name":"languages", + "mode":"repeated" + }, + { + "type":"string", + "name":"utcOffset", + "mode":"nullable" + }, + { + "type":"string", + "name":"link", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"twitter_country_code", + "mode":"nullable" + }, + { + "type":"string", + "name":"displayName", + "mode":"nullable" + }, + { + "type":"string", + "name":"name", + "mode":"nullable" + }, + { + "type":"string", + "name":"link", + "mode":"nullable" + }, + { + "type":"string", + "name":"country_code", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"type", + "mode":"nullable" + }, + { + "type":"float", + "name":"lat", + "mode":"nullable" + }, + { + "type":"float", + "name":"long", + "mode":"nullable" + } + ], + "type":"record", + "name":"geo", + "mode":"nullable" + }, + { + "type":"string", + "name":"twitter_place_type", + "mode":"nullable" + }, + { + "type":"string", + "name":"objectType", + "mode":"nullable" + } + ], + "type":"record", + "name":"location", + "mode":"nullable" + }, + { + "type":"string", + "name":"id", + "mode":"nullable" + }, + { + "type":"string", + "name":"displayName", + "mode":"nullable" + }, + { + "type":"integer", + "name":"listedCount", + "mode":"nullable" + }, + { + "type":"integer", + "name":"followersCount", + "mode":"nullable" + }, + { + "type":"integer", + "name":"favoritesCount", + "mode":"nullable" + }, + { + "type":"integer", + "name":"friendsCount", + "mode":"nullable" + }, + { + "type":"string", + "name":"objectType", + "mode":"nullable" + } + ], + "type":"record", + "name":"actor", + "mode":"nullable" + }, + { + "type":"string", + "name":"twitter_lang", + "mode":"nullable" + }, + { + "type":"string", + "name":"twitter_filter_level", + "mode":"nullable" + }, + { + "type":"string", + "name":"verb", + "mode":"nullable" + }, + { + "type":"string", + "name":"link", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"twitter_country_code", + "mode":"nullable" + }, + { + "type":"string", + "name":"displayName", + "mode":"nullable" + }, + { + "type":"string", + "name":"name", + "mode":"nullable" + }, + { + "type":"string", + "name":"link", + "mode":"nullable" + }, + { + "type":"string", + "name":"country_code", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"type", + "mode":"nullable" + }, + { + "type":"float", + "name":"lat", + "mode":"nullable" + }, + { + "type":"float", + "name":"long", + "mode":"nullable" + } + ], + "type":"record", + "name":"geo", + "mode":"nullable" + }, + { + "type":"string", + "name":"twitter_place_type", + "mode":"nullable" + }, + { + "type":"string", + "name":"objectType", + "mode":"nullable" + } + ], + "type":"record", + "name":"location", + "mode":"nullable" + }, + { + "fields":[ + { + "fields":[ + { + "type":"string", + "name":"source_status_id_str", + "mode":"nullable" + }, + { + "type":"string", + "name":"expanded_url", + "mode":"nullable" + }, + { + "type":"string", + "name":"display_url", + "mode":"nullable" + }, + { + "type":"string", + "name":"url", + "mode":"nullable" + }, + { + "type":"string", + "name":"media_url_https", + "mode":"nullable" + }, + { + "type":"integer", + "name":"source_status_id", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"integer", + "name":"aspect_ratio", + "mode":"repeated" + }, + { + "type":"integer", + "name":"duration_millis", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"url", + "mode":"nullable" + }, + { + "type":"integer", + "name":"bitrate", + "mode":"nullable" + }, + { + "type":"string", + "name":"content_type", + "mode":"nullable" + } + ], + "type":"record", + "name":"variants", + "mode":"repeated" + } + ], + "type":"record", + "name":"video_info", + "mode":"nullable" + }, + { + "type":"string", + "name":"id_str", + "mode":"nullable" + }, + { + "fields":[ + { + "fields":[ + { + "type":"integer", + "name":"h", + "mode":"nullable" + }, + { + "type":"integer", + "name":"w", + "mode":"nullable" + }, + { + "type":"string", + "name":"resize", + "mode":"nullable" + } + ], + "type":"record", + "name":"small", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"integer", + "name":"h", + "mode":"nullable" + }, + { + "type":"integer", + "name":"w", + "mode":"nullable" + }, + { + "type":"string", + "name":"resize", + "mode":"nullable" + } + ], + "type":"record", + "name":"large", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"integer", + "name":"h", + "mode":"nullable" + }, + { + "type":"integer", + "name":"w", + "mode":"nullable" + }, + { + "type":"string", + "name":"resize", + "mode":"nullable" + } + ], + "type":"record", + "name":"medium", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"integer", + "name":"h", + "mode":"nullable" + }, + { + "type":"integer", + "name":"w", + "mode":"nullable" + }, + { + "type":"string", + "name":"resize", + "mode":"nullable" + } + ], + "type":"record", + "name":"thumb", + "mode":"nullable" + } + ], + "type":"record", + "name":"sizes", + "mode":"nullable" + }, + { + "type":"integer", + "name":"indices", + "mode":"repeated" + }, + { + "type":"string", + "name":"type", + "mode":"nullable" + }, + { + "type":"integer", + "name":"id", + "mode":"nullable" + }, + { + "type":"string", + "name":"media_url", + "mode":"nullable" + } + ], + "type":"record", + "name":"media", + "mode":"repeated" + } + ], + "type":"record", + "name":"twitter_extended_entities", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"displayName", + "mode":"nullable" + }, + { + "type":"string", + "name":"link", + "mode":"nullable" + }, + { + "type":"string", + "name":"objectType", + "mode":"nullable" + } + ], + "type":"record", + "name":"provider", + "mode":"nullable" + }, + { + "fields":[ + { + "type":"string", + "name":"type", + "mode":"nullable" + }, + { + "type":"float", + "name":"lat", + "mode":"nullable" + }, + { + "type":"float", + "name":"long", + "mode":"nullable" + } + ], + "type":"record", + "name":"geo", + "mode":"nullable" + }, + { + "type":"string", + "name":"id", + "mode":"nullable" + }, + { + "type":"string", + "name":"objectType", + "mode":"nullable" + } +] \ No newline at end of file diff --git a/utils.py b/utils.py index 3373da9..6391440 100644 --- a/utils.py +++ b/utils.py @@ -159,12 +159,32 @@ def scrub(d): if value is None: del d[key] + # maintain geo with lat/long + elif key == 'geo': + geo = d.get('geo', None) + if geo: + + type = geo.get('type', None) + coordinates = geo.get('coordinates', None) + del geo['coordinates'] + + if type == 'Point': + geo['lat'] = coordinates[0] + geo['long'] = coordinates[1] + elif type == 'Polygon': + geo['lat'] = sum(pair[1] for pair in coordinates[0]) / 4 + geo['long'] = sum(pair[0] for pair in coordinates[0]) / 4 + + print "\t\t\t FOUND", geo + + d['geo'] = geo + # remove other coordinates that are 4-point bounding boxes elif key == 'coordinates': del d[key] - elif key == 'attributes': # in 'place' object - del d[key] elif key == 'bounding_box': # in 'place' object del d[key] + elif key == 'attributes': # in 'place' object + del d[key] elif key == 'retweeted_status': del d[key] elif key == 'created_at':