From 829371a8256b1c335f12c1576ad50834061984bc Mon Sep 17 00:00:00 2001 From: Xinyue Han Date: Sun, 13 Oct 2019 17:56:48 -0700 Subject: [PATCH 1/2] aggregate initial tweet data based on dates --- backend/web/router/tweet_router.py | 45 +++++++++++++- .../src/app/map/layers/fire.tweet.layer.ts | 30 ++++++--- .../map/time-series/time-series.component.ts | 61 +++++++++++++------ .../app/services/map-service/map.service.ts | 5 ++ .../src/app/services/time/time.service.ts | 12 +++- 5 files changed, 124 insertions(+), 29 deletions(-) diff --git a/backend/web/router/tweet_router.py b/backend/web/router/tweet_router.py index 02af014..753aae8 100644 --- a/backend/web/router/tweet_router.py +++ b/backend/web/router/tweet_router.py @@ -57,6 +57,23 @@ def send_live_tweet(): return resp +@bp.route("/tweet-count") +def send_tweet_count_data(): + """ + This func gives all historical tweets objects with id + + :returns: a list of tweet objects, each with time, lat, long, id + """ + resp = make_response( + jsonify({date.isoformat(): count for date, count in + Connection().sql_execute( + "select m.t_date, count(*) from " + "(select r.create_at::timestamp::date as t_date from records r,locations l where r.id=l.id group by(r.create_at)) " + "as m group by m.t_date order by m.t_date")})) + + return resp + + @bp.route("/fire-tweet") def send_fire_tweet_data(): """ @@ -68,7 +85,7 @@ def send_fire_tweet_data(): jsonify([{"create_at": time.isoformat(), "long": lon, "lat": lat, "id": str(id)} for time, lon, lat, _, _, id in Connection().sql_execute( "select r.create_at, l.top_left_long, l.top_left_lat, l.bottom_right_long, l.bottom_right_lat, r.id " - "from records r,locations l where r.id=l.id")])) + "from records r,locations l where r.id=l.id AND create_at>now()-interval '30 day'")])) return resp @@ -145,6 +162,32 @@ def region_tweet(): return resp +@bp.route('/tweet-by-date') +def tweet_by_date(): + """ + tweet count within specific date range + + @:param start-date: ISO string + @:param end-date: ISO string + :return: [ {create_at, id, lat, lon}, ... ] + """ + start_date_str = flask_request.args.get('start-date').split('.')[0][:-3] + end_date_str = flask_request.args.get('end-date').split('.')[0][:-3] + + query = f''' + select r.create_at, r.id, top_left_long, top_left_lat, bottom_right_long, bottom_right_lat + from records r, locations l where r.id = l.id and r.create_at < to_timestamp({end_date_str}) and r.create_at > to_timestamp({start_date_str}) + ''' + + resp = make_response( + jsonify( + [{'create_at':create_at, 'id': id, 'lat': (top_left_lat + bottom_right_lat) / 2, 'long': (top_left_long + bottom_right_long) / 2} + for create_at, id, top_left_long, top_left_lat, bottom_right_long, bottom_right_lat in + Connection.sql_execute(query)])) + + return resp + + @bp.route("/tweet-from-id", methods=['GET']) def tweet_from_id(): """ diff --git a/frontend/src/app/map/layers/fire.tweet.layer.ts b/frontend/src/app/map/layers/fire.tweet.layer.ts index 95bd2d4..4318275 100644 --- a/frontend/src/app/map/layers/fire.tweet.layer.ts +++ b/frontend/src/app/map/layers/fire.tweet.layer.ts @@ -26,7 +26,9 @@ export class FireTweetLayer { constructor(private mainControl, private mapService: MapService, private map, private timeService: TimeService) { // This is the overlay controller defined in constructor + //rewrite this.mapService.getFireTweetData().subscribe(this.tweetDataHandler); + this.map.on('overlayadd', (event) => { if (event.name === 'Fire tweet') { this.map.on('mousemove', e => this.onMapMouseMove(e)); @@ -155,6 +157,7 @@ export class FireTweetLayer { * @param {Object} list of tweet object with geolocation, time of tweet, id of tweet */ this.tweetData = tweets; + console.log(this.tweetData); this.tweetLayer = L.TileLayer.maskCanvas({ radius: 10, useAbsoluteRadius: true, @@ -167,7 +170,7 @@ export class FireTweetLayer { this.tweetData.forEach(tweet => { tempData.push([tweet.lat, tweet.long]); - } + } ); this.tweetLayer.setData(tempData); @@ -182,17 +185,24 @@ export class FireTweetLayer { * Filter out tweets which not satisfy the time range selection; * store qualified tweets with their id in list 'tempDataWithID' for later content display usage */ - const tempData = []; this.tempDataWithID = []; const [startDateInMs, endDateInMs] = this.timeService.getRangeDate(); - this.tweetData.forEach(tweet => { - const time = new Date(tweet.create_at).getTime(); - if (time > startDateInMs && time < endDateInMs) { - tempData.push([tweet.lat, tweet.long]); - this.tempDataWithID.push([tweet.lat, tweet.long, tweet.id]); - } - }); - this.tweetLayer.setData(tempData); + this.timeService.getTweetByDate(startDateInMs, endDateInMs).subscribe((data) => { + console.log(data); + this.tweetLayer.setData(data); + }); + //console.log(startDateInMs); + //console.log(this.timeService.getTweetByDate(startDateInMs, endDateInMs)); + + // this.tweetData.forEach(tweet => { + // const time = new Date(tweet.create_at).getTime(); + // if (time > startDateInMs && time < endDateInMs) { + // tempData.push([tweet.lat, tweet.long]); + // this.tempDataWithID.push([tweet.lat, tweet.long, tweet.id]); + // } + // }); + //console.log(tempData); + //this.tweetLayer.setData(tempData); //draw on twittermap } idOverPoint(x, y) { diff --git a/frontend/src/app/map/time-series/time-series.component.ts b/frontend/src/app/map/time-series/time-series.component.ts index 46b65f6..167d896 100644 --- a/frontend/src/app/map/time-series/time-series.component.ts +++ b/frontend/src/app/map/time-series/time-series.component.ts @@ -31,7 +31,7 @@ export class TimeSeriesComponent implements OnInit { ngOnInit() { /** Subscribe tweet data related to wildfire in service. */ - this.mapService.getFireTweetData().subscribe(data => this.drawTimeSeries(data)); + this.mapService.getDateCountData().subscribe(data => this.drawTimeSeries(data)); } /** @@ -43,25 +43,34 @@ export class TimeSeriesComponent implements OnInit { * @param tweets tweet data crawled using tweet api * */ - drawTimeSeries = (tweets: Tweet[]) => { + drawTimeSeries = (dateAndCount) => { + /** replace */ /** * Refine tweet data to count related to 'wildfire' in each DAY, * storing in charData. */ const chartData = []; - const dailyCount = {}; - for (const tweet of tweets) { - const createAt = tweet.create_at.split('T')[0]; - if (dailyCount.hasOwnProperty(createAt)) { - dailyCount[createAt]++; - } else { - dailyCount[createAt] = 1; - } - } - Object.keys(dailyCount).sort().forEach(key => { - chartData.push([new Date(key).getTime(), dailyCount[key]]); - }); + Object.keys(dateAndCount).forEach(key => { + chartData.push([new Date(key).getTime(), dateAndCount[key]]); + }); + console.log(chartData); + // const chartData = []; + // const dailyCount = {}; + // for (const tweet of tweets) { + // const createAt = tweet.create_at.split('T')[0]; + // if (dailyCount.hasOwnProperty(createAt)) { + // dailyCount[createAt]++; + // } else { + // dailyCount[createAt] = 1; + // } + // } + // Object.keys(dailyCount).sort().forEach(key => { + // chartData.push([new Date(key).getTime(), dailyCount[key]]); + // }); + + /** Plotting format of time-series. */ + console.log("xxxxx"); const timeseries = Highcharts.stockChart('timebar-container', { chart: { height: 150, @@ -123,7 +132,6 @@ export class TimeSeriesComponent implements OnInit { this.hasPlotBand = false; this.timeService.setCurrentDate(undefined); } - }, } }, @@ -161,12 +169,16 @@ export class TimeSeriesComponent implements OnInit { * updating information of date. */ setExtremes: (event) => { - this.timeService.setRangeDate(event.min + this.halfUnit, event.max); + console.log(event.min); + console.log(event.max); + this.timeService.setRangeDate(event.min, event.max); + /** this.timeService.getTweetByDate(event.min, event.max).subscribe((data)=>{});*/ + + $('#report').html('Date Range => ' + 'Start: ' + Highcharts.dateFormat('%Y-%m-%d', event.min) + ', End: ' + Highcharts.dateFormat('%Y-%m-%d', event.max)); $(window).trigger('timeRangeChange'); - } } }, @@ -187,6 +199,7 @@ export class TimeSeriesComponent implements OnInit { * time series click event. */ closestTickNearClick(eventAxis): [number, number, number, any] { + console.log("yyyyy"); const halfUnitDistance = 43200000; const xAxis = eventAxis.axis; const dateClickedInMs = eventAxis.value; @@ -217,6 +230,20 @@ export class TimeSeriesComponent implements OnInit { distanceToTheRight = (xAxis.ordinalPositions[minKey + 1] - xAxis.ordinalPositions[minKey]) / 2; } } + console.log(minValue - distanceToTheLeft); + console.log(minValue); + console.log(distanceToTheRight + minValue); + // this.timeService.setRangeDate(minValue - distanceToTheLeft, distanceToTheRight + minValue); + // $('#report').html('Date Range => ' + + // 'Start: ' + Highcharts.dateFormat('%Y-%m-%d', minValue - distanceToTheLeft) + + // ', End: ' + Highcharts.dateFormat('%Y-%m-%d', distanceToTheRight + minValue)); + // $(window).trigger('timeRangeChange'); + this.timeService.setRangeDate(minValue, (2 * distanceToTheRight) + minValue); + /**this.timeService.getTweetByDate(minValue, (2 * distanceToTheRight) + minValue);*/ + $('#report').html('Date Range => ' + + 'Start: ' + Highcharts.dateFormat('%Y-%m-%d', minValue) + + ', End: ' + Highcharts.dateFormat('%Y-%m-%d', 2 * distanceToTheRight + minValue)); + $(window).trigger('timeRangeChange'); return [minValue - distanceToTheLeft, minValue, distanceToTheRight + minValue, xAxis.ticks[minValue]]; } } diff --git a/frontend/src/app/services/map-service/map.service.ts b/frontend/src/app/services/map-service/map.service.ts index 3da915e..deafe18 100644 --- a/frontend/src/app/services/map-service/map.service.ts +++ b/frontend/src/app/services/map-service/map.service.ts @@ -30,6 +30,11 @@ export class MapService { return this.http.get(`http://${environment.host}:${environment.port}/tweet/fire-tweet`); } + getDateCountData(): Observable { + return this.http.get(`http://${environment.host}:${environment.port}/tweet/tweet-count`); + } + + getWildfirePredictionData(northEastBoundaries, southWestBoundaries, start, end): Observable { return this.http.post(`http://${environment.host}:${environment.port}/wildfire-prediction`, JSON.stringify({ northEast: northEastBoundaries, diff --git a/frontend/src/app/services/time/time.service.ts b/frontend/src/app/services/time/time.service.ts index 9ff4e1b..2c29574 100644 --- a/frontend/src/app/services/time/time.service.ts +++ b/frontend/src/app/services/time/time.service.ts @@ -9,6 +9,10 @@ */ import {Injectable} from '@angular/core'; +import {Observable} from 'rxjs'; +import {environment} from '../../../environments/environment'; +import {HttpClient, HttpParams} from '@angular/common/http'; + // import {Observable, Subject, BehaviorSubject} from 'rxjs'; // import {HttpClient, HttpParams} from '@angular/common/http'; @@ -27,7 +31,8 @@ export class TimeService { private rangeStartDateInMS = new Date().getTime() - 6 * 30 * 24 * 3600 * 1000; private rangeEndDateInMS = new Date().getTime(); - constructor() { + constructor(private http: HttpClient) { + } setCurrentDate(dateInYMD: string): void { @@ -46,5 +51,10 @@ export class TimeService { getRangeDate(): [number, number] { return [this.rangeStartDateInMS, this.rangeEndDateInMS]; } + + getTweetByDate(startDate, endDate): Observable { + return this.http.get(`http://${environment.host}:${environment.port}/tweet/tweet-by-date`, + {params: new HttpParams().set('start-date', startDate ).set('end-date', endDate)}); + } } From 003ef57d39c843232e9bb0b978221615137b24fa Mon Sep 17 00:00:00 2001 From: Yicong-Huang Date: Mon, 14 Oct 2019 07:39:09 -0700 Subject: [PATCH 2/2] fixed getTweetByDate --- frontend/src/app/map/layers/FireTweetLayer.ts | 37 ------------------- .../src/app/map/layers/fire.tweet.layer.ts | 25 ++++++++----- .../src/app/services/time/time.service.ts | 7 ++-- 3 files changed, 19 insertions(+), 50 deletions(-) delete mode 100644 frontend/src/app/map/layers/FireTweetLayer.ts diff --git a/frontend/src/app/map/layers/FireTweetLayer.ts b/frontend/src/app/map/layers/FireTweetLayer.ts deleted file mode 100644 index 73deaeb..0000000 --- a/frontend/src/app/map/layers/FireTweetLayer.ts +++ /dev/null @@ -1,37 +0,0 @@ -import 'leaflet/dist/leaflet.css'; -import {MapService} from '../../services/map-service/map.service'; -import 'leaflet-maskcanvas'; -import 'leaflet-velocity-ts'; -import {Tweet} from '../../models/tweet.model'; - -declare let L; - -export class FireTweetLayer { - private tweetLayer; - private tweetData; - - constructor(private mainControl, private mapService: MapService) { - this.mapService.getFireTweetData().subscribe(this.tweetDataHandler); - } - - - tweetDataHandler = (tweets: Tweet[]) => { - this.tweetLayer = L.TileLayer.maskCanvas({ - radius: 10, - useAbsoluteRadius: true, - color: '#000', - opacity: 1, - noMask: true, - lineColor: '#e25822' - }); - const tempData = []; - this.tweetData = tweets; - tweets.forEach(tweet => { - tempData.push([tweet.lat, tweet.long]); - }); - - this.tweetLayer.setData(tempData); - this.mainControl.addOverlay(this.tweetLayer, 'Fire tweet'); - - } -} diff --git a/frontend/src/app/map/layers/fire.tweet.layer.ts b/frontend/src/app/map/layers/fire.tweet.layer.ts index 4318275..5cb86cf 100644 --- a/frontend/src/app/map/layers/fire.tweet.layer.ts +++ b/frontend/src/app/map/layers/fire.tweet.layer.ts @@ -26,7 +26,7 @@ export class FireTweetLayer { constructor(private mainControl, private mapService: MapService, private map, private timeService: TimeService) { // This is the overlay controller defined in constructor - //rewrite + // rewrite this.mapService.getFireTweetData().subscribe(this.tweetDataHandler); this.map.on('overlayadd', (event) => { @@ -170,7 +170,7 @@ export class FireTweetLayer { this.tweetData.forEach(tweet => { tempData.push([tweet.lat, tweet.long]); - } + } ); this.tweetLayer.setData(tempData); @@ -187,12 +187,17 @@ export class FireTweetLayer { */ this.tempDataWithID = []; const [startDateInMs, endDateInMs] = this.timeService.getRangeDate(); - this.timeService.getTweetByDate(startDateInMs, endDateInMs).subscribe((data) => { - console.log(data); - this.tweetLayer.setData(data); - }); - //console.log(startDateInMs); - //console.log(this.timeService.getTweetByDate(startDateInMs, endDateInMs)); + this.timeService.getTweetByDate(startDateInMs, endDateInMs).subscribe(tweets => { + const tempData = []; + tweets.forEach(tweet => { + tempData.push([tweet.lat, tweet.long]); + } + ); + this.tweetLayer.setData(tempData); + } + ); + // console.log(startDateInMs); + // console.log(this.timeService.getTweetByDate(startDateInMs, endDateInMs)); // this.tweetData.forEach(tweet => { // const time = new Date(tweet.create_at).getTime(); @@ -201,8 +206,8 @@ export class FireTweetLayer { // this.tempDataWithID.push([tweet.lat, tweet.long, tweet.id]); // } // }); - //console.log(tempData); - //this.tweetLayer.setData(tempData); //draw on twittermap + // console.log(tempData); + // this.tweetLayer.setData(tempData); //draw on twittermap } idOverPoint(x, y) { diff --git a/frontend/src/app/services/time/time.service.ts b/frontend/src/app/services/time/time.service.ts index 2c29574..983a1eb 100644 --- a/frontend/src/app/services/time/time.service.ts +++ b/frontend/src/app/services/time/time.service.ts @@ -12,6 +12,7 @@ import {Injectable} from '@angular/core'; import {Observable} from 'rxjs'; import {environment} from '../../../environments/environment'; import {HttpClient, HttpParams} from '@angular/common/http'; +import {Tweet} from "../../models/tweet.model"; // import {Observable, Subject, BehaviorSubject} from 'rxjs'; // import {HttpClient, HttpParams} from '@angular/common/http'; @@ -52,9 +53,9 @@ export class TimeService { return [this.rangeStartDateInMS, this.rangeEndDateInMS]; } - getTweetByDate(startDate, endDate): Observable { - return this.http.get(`http://${environment.host}:${environment.port}/tweet/tweet-by-date`, - {params: new HttpParams().set('start-date', startDate ).set('end-date', endDate)}); + getTweetByDate(startDate, endDate): Observable { + return this.http.get(`http://${environment.host}:${environment.port}/tweet/tweet-by-date`, + {params: new HttpParams().set('start-date', startDate).set('end-date', endDate)}); } }