* add formrequest version of spider and checker

dynamic_scraper/spiders/django_checker.py

@@ -4,6 +4,7 @@
 settings = get_project_settings()
 from scrapy.exceptions import CloseSpider
 from scrapy.selector import HtmlXPathSelector
+from scrapy.http import Request, FormRequest
 from scrapy.xlib.pydispatch import dispatcher
 
 from dynamic_scraper.spiders.django_base_spider import DjangoBaseSpider
@@ -99,3 +100,25 @@ def parse(self, response):
         else:
             self.log("XPath result string not found. Item kept.", log.INFO)
             return
+
+class FormRequestDjangoChecker(DjangoChecker):
+
+    def __init__(self, *args, **kwargs):
+        super(FormRequestDjangoChecker, self).__init__(self, *args, **kwargs)
+
+        assert 'username' in kwargs
+        assert 'password' in kwargs
+        assert 'username_form' in kwargs
+        assert 'password_form' in kwargs
+
+        self.username, self.password, self.username_form, self.password_form \
+            = kwargs['username'], kwargs['password'], \
+              kwargs['username_form'], kwargs['password_form']
+
+    def parse(self, response):
+        return FormRequest.from_response(
+            response,
+            formdata={self.username_form: self.username,
+	              self.password_form: self.password},
+            callback=super(FormRequestDjangoChecker, self).parse
+        )

dynamic_scraper/spiders/django_spider.py

@@ -2,7 +2,7 @@
 
 from scrapy import log
 from scrapy.selector import HtmlXPathSelector, XmlXPathSelector
-from scrapy.http import Request
+from scrapy.http import Request, FormRequest
 from scrapy.contrib.loader import XPathItemLoader
 from scrapy.contrib.loader.processor import TakeFirst
 from scrapy.exceptions import CloseSpider
@@ -239,3 +239,24 @@ def parse(self, response):
             else:
                 self.log("Detail page url elem could not be read!", log.ERROR)
 
+class FormRequestDjangoSpider(DjangoSpider):
+
+    def __init__(self, *args, **kwargs):
+        super(FormRequestDjangoSpider, self).__init__(self, *args, **kwargs)
+
+        assert 'username' in kwargs
+        assert 'password' in kwargs
+        assert 'username_form' in kwargs
+        assert 'password_form' in kwargs
+
+        self.username, self.password, self.username_form, self.password_form \
+            = kwargs['username'], kwargs['password'], \
+              kwargs['username_form'], kwargs['password_form']
+
+    def parse(self, response):
+        return FormRequest.from_response(
+            response,
+            formdata={self.username_form: self.username,
+	              self.password_form: self.password},
+            callback=super(FormRequestDjangoSpider, self).parse
+        )