allow backfill of tweets on table and rule creation

app.py

@@ -54,7 +54,7 @@ class TableList(webapp2.RequestHandler):
 
     def get(self):
 
-        template_data = {"projectId" : config.PROJECT_ID}
+        template_data = {'searchDays' : SEARCH_DAYS, "projectId" : config.PROJECT_ID}
         self.response.out.write(JINJA.get_template('table_list.html').render(template_data))
 
 class ApiTableList(webapp2.RequestHandler):
@@ -268,13 +268,18 @@ class ApiRuleTest(webapp2.RequestHandler):
     def get(self):
 
         rule = self.request.get("rule")
+        days = SEARCH_DAYS
+        try:
+            days = int(self.request.get("days", SEARCH_DAYS))
+        except:
+            pass
 
         if not rule:
             raise Exception("missing parameter")
 
         g = Utils.get_gnip()
         end = datetime.now()
-        start = end - timedelta(days=SEARCH_DAYS)
+        start = end - timedelta(days=days)
         timeline = g.query(rule, 0, record_callback=None, use_case="timeline", start=start, end=end, count_bucket="day")
         timeline = json.loads(timeline)
 
@@ -293,8 +298,18 @@ def get(self):
 
         rule = self.request.get("rule", None)
         table = self.request.get("table", None)
+        days = SEARCH_DAYS
+        try:
+            days = int(self.request.get("days", SEARCH_DAYS))
+
+            print "DAYS E", days
 
-        self.enqueue(rule, table, page_next=None, page_count=0, count_total=0)
+        except:
+            pass
+
+        print "DAYS F", days
+
+        self.enqueue(rule, table, days, page_next=None, page_count=0, count_total=0)
 
         response = {
             "enqueued" : True
@@ -306,9 +321,18 @@ def post(self):
 
         rule = self.request.get("rule", None)
         table_fqdn = self.request.get("table", None)
+
+        print table_fqdn
+
         (dataset, table) = Utils.parse_bqid(table_fqdn)  
         tag = Utils.make_tag(dataset, table)
-
+
+        days = SEARCH_DAYS
+        try:
+            days = int(self.request.get("days", SEARCH_DAYS))
+        except:
+            pass
+
         page_next = self.request.get("page_next", None)
         page_count = self.request.get("page_count", None)
         count_total = int(self.request.get("count_total", 0))
@@ -318,7 +342,7 @@ def post(self):
             page_count = int(page_count) + 1
 
         end = datetime.now()
-        start = end - timedelta(days=SEARCH_DAYS)
+        start = end - timedelta(days=days)
 
         # for logging purposes, show the estimate 
         if not page_next:
@@ -353,7 +377,7 @@ def post(self):
 
         page_next = g.rule_payload.get("next", None)
         if page_next:
-            self.enqueue(rule, table_fqdn, page_next, page_count=page_count, count_total=count_total)
+            self.enqueue(rule, table_fqdn, days, page_next, page_count=page_count, count_total=count_total)
 
         response = {
             "completed" : True
@@ -363,11 +387,12 @@ def post(self):
 
     # enqueue a pagination task. needed because AppEngine limits tasks to < 10 minutes, 
     # so we can't run long-running tasks
-    def enqueue(self, rule, table, page_next=None, page_count=0, count_total=0):
+    def enqueue(self, rule, table, days, page_next=None, page_count=0, count_total=0):
 
         params = {
             "rule": rule,
             "table": table,
+            "days": days,
             "page_count": page_count,
             "count_total": count_total 
         }

static/js/script.js

@@ -260,7 +260,8 @@ var RulePage = {
 		$(document.body).on("click", ".rule_test", function(){
 			var rule = $("#rule_text").val();
 			var tag = $("#rule_tag").val();
-			RulePage.test(rule, RulePage.test_callback)
+			var days = $("#backfill_days").val();
+			RulePage.test(rule, days, RulePage.test_callback)
 		});
 
 		// any change to rule results in need for re-test
@@ -270,12 +271,29 @@ var RulePage = {
 		});
 
 		$(document.body).on("click", ".rule_add", function(){
+
 			var rule = $("#rule_text").val();
 			var tag = $("#rule_tag").val();
+			var backfill_days = $("#backfill_days").val();
 
 			RulePage.add(rule, tag, function(response){
-				$('#ruleModal').modal('hide');
-				RulePage.list(table_id);
+
+				if ($("#rule_import").is(":checked")){
+
+					RulePage.backfill(rule, tag, backfill_days, function(response){
+
+						$('#ruleModal').modal('hide');
+						RulePage.list(table_id);
+
+					});
+
+				} else {
+
+					$('#ruleModal').modal('hide');
+					RulePage.list(table_id);
+
+				}
+
 			});
 
 		});
@@ -288,22 +306,9 @@ var RulePage = {
 	init_import_count : function(rule_text_field){
 
 		$("#rule_import_count").hide();
-	    $('#rule_import').change(function() {
-
-	    	var rule = $(rule_text_field).val();
-
-	    	if (!rule){
-	    		alert("Please enter a rule to calculate volume of tweets.");
-	    		$(this).attr("checked", false);
-	    		return false;
-	    	}
-
-	        if($(this).is(":checked")) {
-		    	RulePage.test(rule, RulePage.test_callback);
-	        } else {
-	        	$("#rule_import_count").hide();
-	        }
-	    });
+//	    $('#rule_import').change(function() {
+//	    	$("#rule_import_days").prop('disabled', !$(this).is(":checked"));
+//	    });
 
 	},
 
@@ -315,11 +320,16 @@ var RulePage = {
 		 Page.add("/api/rule/add", params, callback);
 	},
 
-	test : function(rule, success, error){
+	test : function(rule, days, success, error){
+
+		if (!rule){
+			alert("Please enter a rule.");
+			return false;
+		}
 
     	$("#rule_import_count").fadeIn();
     	$("#rule_import_loading").show();
-    	$("#rule_import_text").html("Calculating volume of tweets over last " + RulePage.DAYS + " days...")
+    	$("#rule_import_text").html("Calculating volume of tweets over last " + days + " days...")
 
     	if (!error){
     		error = function (request, status, error){
@@ -329,7 +339,8 @@ var RulePage = {
     	}
 
 		 var params = {
-			'rule': rule
+			'rule': rule,
+			'days': days
 		 }
 		 $.ajax({
 			type : "GET",
@@ -348,10 +359,11 @@ var RulePage = {
 		$("#rule_add").prop("disabled", false);
 	},
 
-	backfill : function(rule, table, callback){
+	backfill : function(rule, table, days, callback){
 		 var params = {
 			'rule': rule,
-			'table': table
+			'table': table,
+			'days': days
 		 }
 		 $.ajax({
 			type : "GET",
@@ -392,12 +404,37 @@ var TablePage = {
 			var table = $("#table_name").val();
 			var type = $("#table_type").val();
 			var rules = $("#table_rules").val();
+			var backfill_days = $("#backfill_days").val();
 			var imprt = $("#table_imprt").val();
 
+			if (!table){
+				alert("Please enter a table name.");
+				return;
+			}
+
 			TablePage.add(dataset, table, type, rules, imprt, function(response){
-				$('#ruleModal').modal('hide');
-				$("#datasets").html("");
-				TablePage.list();
+
+				if ($("#rule_import").is(":checked")){
+
+					// BUGBUG: dataset has '.' at end, and it shouldn't
+					var rule_tag = dataset + table;
+
+					RulePage.backfill(rules, rule_tag, backfill_days, function(response){
+
+						$('#ruleModal').modal('hide');
+						$("#datasets").html("");
+						TablePage.list();
+
+					});
+
+				} else {
+
+					$('#ruleModal').modal('hide');
+					$("#datasets").html("");
+					TablePage.list();
+
+				}
+
 			});
 		});
 
@@ -409,7 +446,8 @@ var TablePage = {
 
 		$(document.body).on("click", "#rule_test", function(){
 			var rule = $("#table_rules").val();
-			RulePage.test(rule, TablePage.test_callback)
+			var days = $("#backfill_days").val();
+			RulePage.test(rule, days, TablePage.test_callback)
 		});
 
 		$(document.body).on("change", "#table_type", function(){

templates/rule_add_partial.html

@@ -23,16 +23,18 @@ <h4 class="modal-title" id="ruleModalLabel">Add rule</h4>
 				</select>
 		    </div>
 		  </div>
-<!--
 		  <div class="form-group">
-		    <label for="" class="col-sm-2 control-label"></label>
-		    <div class="col-sm-10 checkbox">
+		    <label class="col-sm-2 control-label"></label>
+		    <div class="col-sm-5 checkbox">
 		    	<label for="rule_import">
-		      		<input type="checkbox" id="rule_import"> Import last 7 days of data
+		      		<input type="checkbox" id="rule_import"> Import older tweets
 	      		</label>
 		    </div>
+		    <div class="col-sm-2">
+		    	<input type="text" class="form-control" size="2" maxlength="2" value="{{searchDays}}" id="backfill_days">
+		    </div>
+		    <p class="col-sm-2 form-control-static">days</p>
 		  </div>
--->
 		  <hr>
 		  <div class="form-group" id="rule_import_count">
 		    <div class="col-sm-12">

templates/table_list.html

@@ -72,16 +72,18 @@ <h4 class="modal-title" id="ruleModalLabel">Create table</h4>
 		      <textarea class="form-control" id="table_rules" placeholder="" rows="4"></textarea>
 		    </div>
 		  </div>
-<!--
 		  <div class="form-group">
-		    <label for="" class="col-sm-2 control-label"></label>
-		    <div class="col-sm-10 checkbox">
+		    <label class="col-sm-2 control-label"></label>
+		    <div class="col-sm-5 checkbox">
 		    	<label for="rule_import">
-		      		<input type="checkbox" id="rule_import"> Import last 7 days of data
+		      		<input type="checkbox" id="rule_import"> Import older tweets
 	      		</label>
 		    </div>
+		    <div class="col-sm-2">
+		    	<input type="text" class="form-control" size="2" maxlength="2" value="{{searchDays}}" id="backfill_days">
+		    </div>
+		    <p class="col-sm-2 form-control-static">days</p>
 		  </div>
--->
 		  <hr>
 		  <div class="form-group" id="rule_import_count">
 		    <div class="col-sm-12">
@@ -132,7 +134,8 @@ <h4 class="modal-title" id="ruleModalLabel">Create table</h4>
 
 $(document).ready(function(){
 
-	  TablePage.init_list();
+	RulePage.init({{searchDays}})
+	TablePage.init_list();
 
 })