-
Notifications
You must be signed in to change notification settings - Fork 2
/
scraper.py
77 lines (55 loc) · 2.41 KB
/
scraper.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
import re
import scraperwiki
import requests
from lxml.html.soupparser import fromstring
from time import sleep
month = "Oct 17"
laurlroot = "http://planning.lewisham.gov.uk"
def search(mth):
# session = requests.Session()
request_data = {"month": mth, "dateType": "DC_Validated" , "searchType": "Application" }
sleep(2)
# result = session.post(laurlroot + '/online-applications/monthlyListResults.do?action=firstPage', request_data)
result = requests.post(laurlroot + '/online-applications/monthlyListResults.do?action=firstPage', request_data)
if not result:
print "No result returned"
return
result_dom = fromstring(result.content)
applications = result_dom.xpath("//li[@class='searchresult']")
print len(applications)
for application in applications:
link = "".join(application.xpath('a/@href')).strip()
description = "".join(application.xpath('a/text()')).strip()
address = "".join(application.xpath('p[@class="address"]/text()')).strip()
meta = "".join(application.xpath('p[@class="metaInfo"]/text()')).strip()
print link, address
# GET on the url with searchCriteria.page=N ...
# result = session.get(laurlroot + '/online-applications/pagedSearchResults.do?action=page&searchCriteria.page=2')
# result_dom = fromstring(result.content)
# applications = result_dom.xpath("//li[@class='searchresult']")
# print len(applications)
#searchCriteria.page="n"
#action" value="page"
# this is a guess at what data to post
# request_data = {"month": mth,
# "dateType": "DC_Validated" ,
# "searchType": "Application",
# "searchCriteria.page": "2" ,
# "action": "page",
# "searchCriteria.resultsPerPage": "5"}
# sleep(2)
# result = session.post(laurlroot + '/online-applications/pagedSearchResults.do', request_data)
# if not result:
# print "No result returned for second post"
# return
# result_dom = fromstring(result.content)
# applications = result_dom.xpath("//li[@class='searchresult']")
# print len(applications)
# for application in applications:
# link = "".join(application.xpath('a/@href')).strip()
# description = "".join(application.xpath('a/text()')).strip()
# address = "".join(application.xpath('p[@class="address"]/text()')).strip()
# meta = "".join(application.xpath('p[@class="metaInfo"]/text()')).strip()
# print link, address
print "Starting run"
search(month)