baselineuc.py

# -*- coding: utf-8 -*-
"""baselineuc.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1RZdpcKEQtv3388BueFDigdnJ5174FIBM
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import requests
!pip install openrouteservice
import openrouteservice as opn
from openrouteservice.directions import directions
from openrouteservice import convert
!pip install folium
import folium
import pandas as pd
from datetime import timedelta
import numpy.fft as fft
from statsmodels.tsa.stattools import pacf
from scipy import signal
import warnings
!pip install geopandas
import geopandas as gpd
warnings.filterwarnings('ignore')

csv_url="http://data.insideairbnb.com/ireland/2020-10-16/visualisations/listings.csv"
req = requests.get(csv_url)
url_content = req.content
csv_file = open("listings_ireland.csv", 'wb')
csv_file.write(url_content)
csv_file.close()

csv_url2="http://data.insideairbnb.com/ireland/2020-10-16/visualisations/neighbourhoods.geojson"
req = requests.get(csv_url2)
url_content = req.content
csv_file = open("geo_ireland.geojson", 'wb')
csv_file.write(url_content)
csv_file.close()

listings=pd.read_csv("listings_ireland.csv")
#listings["price_pr"]=listings["price"]/listings["price_pr"]
listings

geo=gpd.read_file("geo_ireland.geojson")
geo

plt.hist(listings.price[listings.price<500],bins=100)
plt.show()

listings_reasonable=listings[listings.price<300]
listings_reasonable.plot.scatter('latitude', 'longitude', c='price', colormap='jet')

from shapely.geometry import Point, shape

locs_geometry = [Point(xy) for xy in zip(listings.longitude,
                                         listings.latitude)]
crs = {'init': 'epsg:4326'}
# Coordinate Reference Systems, "epsg:4326" is a common projection of WGS84 Latitude/Longitude
locs_gdf = gpd.GeoDataFrame(listings, crs=crs, geometry=locs_geometry)
locs_map = folium.Map(location=[np.mean(listings.latitude),np.mean(listings.longitude)],
                      zoom_start=7, tiles='cartodbpositron')

feature_ea = folium.FeatureGroup(name='Entire home/apt')
feature_pr = folium.FeatureGroup(name='Private room')
feature_sr = folium.FeatureGroup(name='Shared room')

for i, v in locs_gdf.iterrows():
    popup = """
    Location id : <b>%s</b><br>
    Room type : <b>%s</b><br>
    Neighbourhood : <b>%s</b><br>
    Price : <b>%d</b><br>
    """ % (v['id'], v['room_type'], v['neighbourhood'], v['price'])
    
    if v['room_type'] == 'Entire home/apt':
        folium.CircleMarker(location=[v['latitude'], v['longitude']],
                            radius=1,
                            tooltip=popup,
                            color='#FFBA00',
                            fill_color='#FFBA00',
                            fill=True).add_to(feature_ea)
    elif v['room_type'] == 'Private room':
        folium.CircleMarker(location=[v['latitude'], v['longitude']],
                            radius=1,
                            tooltip=popup,
                            color='#087FBF',
                            fill_color='#087FBF',
                            fill=True).add_to(feature_pr)
    elif v['room_type'] == 'Shared room':
        folium.CircleMarker(location=[v['latitude'], v['longitude']],
                            radius=1,
                            tooltip=popup,
                            color='#FF0700',
                            fill_color='#FF0700',
                            fill=True).add_to(feature_sr)

feature_ea.add_to(locs_map)
feature_pr.add_to(locs_map)
feature_sr.add_to(locs_map)
folium.LayerControl(collapsed=False).add_to(locs_map)
path='map_scatter.html'
locs_map.save(path)
locs_map

import folium.plugins
cluster_map = folium.Map(location=[np.mean(listings.latitude),np.mean(listings.longitude)],
                         zoom_start=7, tiles='cartodbpositron')

marker_cluster = folium.plugins.MarkerCluster().add_to(cluster_map)
for i, v in locs_gdf.iterrows():
    popup = """
    Location id : <b>%s</b><br>
    Room type : <b>%s</b><br>
    Neighbourhood : <b>%s</b><br>
    Price : <b>%d</b><br>
    """ % (v['id'], v['room_type'], v['neighbourhood'], v['price'])
    
    if v['room_type'] == 'Entire home/apt':
        folium.CircleMarker(location=[v['latitude'], v['longitude']],
                            radius=3,
                            tooltip=popup,
                            color='#FFBA00',
                            fill_color='#FFBA00',
                            fill=True).add_to(marker_cluster)
    elif v['room_type'] == 'Private room':
        folium.CircleMarker(location=[v['latitude'], v['longitude']],
                            radius=3,
                            tooltip=popup,
                            color='#087FBF',
                            fill_color='#087FBF',
                            fill=True).add_to(marker_cluster)
    elif v['room_type'] == 'Shared room':
        folium.CircleMarker(location=[v['latitude'], v['longitude']],
                            radius=3,
                            tooltip=popup,
                            color='#FF0700',
                            fill_color='#FF0700',
                            fill=True).add_to(marker_cluster)
#cluster_map

nbh_count_df = listings.groupby('neighbourhood')['id'].unique().reset_index()
nbh_count_df['id']=[1.0*len(x) for x in nbh_count_df["id"]]
nbh_count_df.rename(columns={'id':'nba'}, inplace=True)
#nbh_count_df['nba']=[1.0*len(x) for x in nbh_count_df["id"]]
nbh_geo_count_df = pd.merge(geo, nbh_count_df, on='neighbourhood')
nbh_geo_count_df['QP'] = nbh_count_df['nba']/nbh_count_df['nba'].sum()
nbh_geo_count_df['QP_str'] = nbh_geo_count_df['QP'].apply(lambda x : str(round(x*100, 1)) + '%')
print(nbh_geo_count_df)

from branca.colormap import linear
nbh_count_colormap = linear.YlGnBu_09.scale(min(nbh_geo_count_df['QP']),
                                            max(nbh_geo_count_df['QP']))

nbh_locs_map = folium.Map(location=[np.mean(listings.latitude),np.mean(listings.longitude)],
                          zoom_start = 7, tiles='cartodbpositron')

style_function = lambda x: {
    'fillColor': nbh_count_colormap(x['properties']['QP']),
    'color': 'black',
    'weight': 1.5,
    'fillOpacity': 0.7
}

folium.GeoJson(
    nbh_geo_count_df,
    style_function=style_function,
    tooltip=folium.GeoJsonTooltip(
        fields=['neighbourhood', 'QP'],
        aliases=['Neighbourhood', 'fraction of living'],
        localize=True
    )
).add_to(nbh_locs_map)

nbh_count_colormap.add_to(nbh_locs_map)
nbh_count_colormap.caption = 'Airbnb location amount'
nbh_count_colormap.add_to(nbh_locs_map)

path='map_listings.html'
nbh_locs_map.save(path)
#nbh_locs_map

nbh_count_df = listings.groupby('neighbourhood')['price'].median().reset_index()
#nbh_count_df.rename(columns={'price':'nb'}, inplace=True)
nbh_geo_count_df = pd.merge(geo, nbh_count_df, on='neighbourhood')
nbh_geo_count_df['QP'] = nbh_geo_count_df['price']
nbh_geo_count_df['QP_str'] = nbh_geo_count_df['QP'].apply(lambda x : str(round(x*100, 1)) + '%')
print(nbh_geo_count_df)

from branca.colormap import linear
nbh_count_colormap = linear.YlGnBu_09.scale(min(nbh_count_df['price']),
                                            max(nbh_count_df['price']))

nbh_locs_map = folium.Map(location=[np.mean(listings.latitude),np.mean(listings.longitude)],
                          zoom_start = 7, tiles='cartodbpositron')

style_function = lambda x: {
    'fillColor': nbh_count_colormap(x['properties']['price']),
    'color': 'black',
    'weight': 1.5,
    'fillOpacity': 0.7
}

folium.GeoJson(
    nbh_geo_count_df,
    style_function=style_function,
    tooltip=folium.GeoJsonTooltip(
        fields=['neighbourhood', 'price'],
        aliases=['Neighbourhood', 'median price'],
        localize=True
    )
).add_to(nbh_locs_map)

nbh_count_colormap.add_to(nbh_locs_map)
nbh_count_colormap.caption = 'Airbnb location amount'
nbh_count_colormap.add_to(nbh_locs_map)

path='map.html'
nbh_locs_map.save(path)
#nbh_locs_map

"""# Baseline models"""

#BASELINE 1/3 DECISION TREE

import pandas as pd
import numpy as np
df2=pd.read_csv('data - data.csv')
from sklearn import tree
from sklearn.ensemble import RandomForestRegressor

errorlist=[]
r2scorelist=[]
for k in range (100):
  df2=df2.sample(frac=1)
  X, y = df2[["neighbourhood_group","latitude","longitude","number_of_reviews"]],df2["price"]
  X.neighbourhood_group=pd.Categorical(X.neighbourhood_group)
  X.neighbourhood_group=X.neighbourhood_group.cat.codes
  l=int(len(X)*0.9)
  X_train,y_train=X[:l],y[:l]
  X_test,y_test=X[l:],y[l:]
  clf = tree.DecisionTreeRegressor(random_state=0)
  clf.fit(X_train, y_train)
  y_pred=clf.predict(X_test)
  #tree.plot_tree(clf)
  from sklearn.metrics import mean_absolute_error,mean_squared_error,r2_score
  errorlist.append(mean_squared_error(y_test,y_pred, squared=False))
  r2scorelist.append(r2_score(y_test,y_pred))

print(np.average(errorlist),np.average(r2scorelist))

#BASELINE 2/3 RANDOM FOREST

import pandas as pd
import numpy as np
from sklearn import tree
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_absolute_error,mean_squared_error,r2_score
#lets build a tree model to predict prices
df2=pd.read_csv('data - data.csv')


errorlist=[]
r2scorelist=[]
for k in range (100):
  df2=df2.sample(frac=1)
  X, y = df2[["neighbourhood","latitude","longitude","number_of_reviews"]],df2["price"]
  X.neighbourhood=pd.Categorical(X.neighbourhood)
  X.neighbourhood=X.neighbourhood.cat.codes
  l=int(len(X)*0.9)
  X_train,y_train=X[:l],y[:l]
  X_test,y_test=X[l:],y[l:]
  clf = RandomForestRegressor(n_estimators=256,random_state=0)
  clf.fit(X_train, y_train)
  y_pred=clf.predict(X_test)
  #tree.plot_tree(clf)
  
  errorlist.append(mean_squared_error(y_test,y_pred, squared=False))
  r2scorelist.append(r2_score(y_test,y_pred))

print(np.average(errorlist),np.average(r2scorelist))

#BASELINE 3/3 POLYNOMIAL REGRESSION
df4=pd.read_csv('data - data.csv')
from sklearn import linear_model
from sklearn.preprocessing import PolynomialFeatures
errorlist=[]
r2scorelist=[]
for k in range (100):
  df4=df4.sample(frac=1)
  X, y = df4[["availability_365","minimum_nights","neighbourhood","latitude","longitude","number_of_reviews","calculated_host_listings_count"]],df4["price"]
  X.neighbourhood=pd.Categorical(X.neighbourhood)
  X.neighbourhood=X.neighbourhood.cat.codes
  poly = PolynomialFeatures(2)
  X=poly.fit_transform(X)
  l=int(len(X)*0.9)
  X_train,y_train=X[:l],y[:l]
  X_test,y_test=X[l:],y[l:]
  clf = linear_model.Ridge(random_state=0)
  clf.fit(X_train, y_train)
  y_pred=clf.predict(X_test)
  errorlist.append(mean_squared_error(y_test,y_pred, squared=False))
  r2scorelist.append(r2_score(y_test,y_pred))


print(np.average(errorlist),np.average(r2scorelist))

"""# Experimental models"""

#CLEANLISTING 1/3 DECISION TREE
import pandas as pd
import numpy as np
df=pd.read_csv('data5labels.csv')
from sklearn import tree
from sklearn.metrics import mean_absolute_error,mean_squared_error,r2_score

megalist=[]
r2megalist=[]
for p in range (500):

  errorlist=[]
  r2scorelist=[]
  for k in range (1):
    df=df.sample(frac=1)
    X, y = df[["neighbourhood","latitude","longitude","number_of_reviews","commonlabels"]],df["price"]
    X.commonlabels=pd.Categorical(X.commonlabels)
    X.commonlabels=X.commonlabels.cat.codes
    X.neighbourhood=pd.Categorical(X.neighbourhood)
    X.neighbourhood=X.neighbourhood.cat.codes
    l=int(len(X)*0.9)
    X_train,y_train=X[:l],y[:l]
    X_test,y_test=X[l:],y[l:]
    clf = tree.DecisionTreeRegressor(random_state=None)
    clf.fit(X_train, y_train)
    y_pred=clf.predict(X_test)
    errorlist.append(mean_squared_error(y_test,y_pred, squared=False))
    r2scorelist.append(r2_score(y_test,y_pred))

    megalist.append(np.average(errorlist))
    r2megalist.append(np.average(r2scorelist))

print(np.average(megalist),np.average(r2megalist))
print(np.min(megalist))
print(np.max(r2megalist))

#CLEANLISTING 2/3 RANDOM FOREST
#lets build a tree model to predict prices
df3=pd.read_csv('data3label.csv')
pd.options.mode.chained_assignment = None
import matplotlib.pyplot as plt
from sklearn import tree
from sklearn.ensemble import RandomForestRegressor
errorlist=[]
r2scorelist=[]
for k in range (100):
  df3=df3.sample(frac=1)
  X, y = df3[["neighbourhood","latitude","longitude","number_of_reviews","commonlabels"]],df3["price"]
  X.commonlabels=pd.Categorical(X.commonlabels)
  X.commonlabels=X.commonlabels.cat.codes
  X.neighbourhood=pd.Categorical(X.neighbourhood)
  X.neighbourhood=X.neighbourhood.cat.codes
  l=int(len(X)*0.9)
  X_train,y_train=X[:l],y[:l]
  X_test,y_test=X[l:],y[l:]
  clf = RandomForestRegressor(n_estimators=256,random_state=None)
  clf.fit(X_train, y_train)
  y_pred=clf.predict(X_test)
  errorlist.append(mean_squared_error(y_test,y_pred, squared=False))
  r2scorelist.append(r2_score(y_test,y_pred))

print(np.average(errorlist),np.average(r2scorelist))

#CLEANLISTING 3/3 POLYNOMIAL REGRESSION
import pandas as pd
import numpy as np
from sklearn.metrics import mean_absolute_error,mean_squared_error,r2_score
df4=pd.read_csv('data5labels.csv')
from sklearn import linear_model
from sklearn.preprocessing import PolynomialFeatures
pd.options.mode.chained_assignment = None
megalist=[]
r2megalist=[]
for p in range (500):
  errorlist=[]
  r2scorelist=[]
  df4=df4.sample(frac=1)
  for k in range (1):
    X, y = df4[["neighbourhood","latitude","longitude","number_of_reviews","commonlabels"]],df4["price"]
    X.commonlabels=pd.Categorical(X.commonlabels)
    X.commonlabels=X.commonlabels.cat.codes
    X.neighbourhood=pd.Categorical(X.neighbourhood)
    X.neighbourhood=X.neighbourhood.cat.codes
    poly = PolynomialFeatures(2)
    X=poly.fit_transform(X)
    l=int(len(X)*0.9)
    X_train,y_train=X[:l],y[:l]
    X_test,y_test=X[l:],y[l:]
    clf = linear_model.Ridge(random_state=None)
    clf.fit(X_train, y_train)
    y_pred=clf.predict(X_test)
    errorlist.append(mean_squared_error(y_test,y_pred, squared=False))
    r2scorelist.append(r2_score(y_test,y_pred))


  megalist.append(np.average(errorlist))
  r2megalist.append(np.average(r2scorelist))

print(np.average(megalist),np.average(r2megalist))
print(np.min(megalist))
print(np.max(r2megalist))



errorlist

X

from sklearn.neighbors import KNeighborsClassifier

df5=pd.read_csv('data - data.csv')
errorlist=[]
r2scorelist=[]
for k in range (100):
  df5=df5.sample(frac=1)
  X, y = df5[["neighbourhood","latitude","longitude","number_of_reviews","commonlabels"]],df5["price"]
  X.commonlabels=pd.Categorical(X.commonlabels)
  X.commonlabels=X.commonlabels.cat.codes
  X.neighbourhood=pd.Categorical(X.neighbourhood)
  X.neighbourhood=X.neighbourhood.cat.codes
  l=int(len(X)*0.9)
  X_train,y_train=X[:l],y[:l]
  X_test,y_test=X[l:],y[l:]
  knn = KNeighborsClassifier()
  knn.fit(X_train, y_train)

  y_pred= knn.predict(X_test)
  errorlist.append(mean_squared_error(y_test,y_pred, squared=False))
  r2scorelist.append(r2_score(y_test,y_pred))

print(np.average(errorlist),np.average(r2scorelist))

from sklearn.neighbors import KNeighborsClassifier

df5=pd.read_csv('data - data.csv')
errorlist=[]
r2scorelist=[]
for k in range (1):
  df5=df5.sample(frac=1)
  X, y = df5[["neighbourhood","latitude","longitude","number_of_reviews","commonlabels"]],df5["price"]
  X.commonlabels=pd.Categorical(X.commonlabels)
  X.commonlabels=X.commonlabels.cat.codes
  X.neighbourhood=pd.Categorical(X.neighbourhood)
  X.neighbourhood=X.neighbourhood.cat.codes
  l=int(len(X)*0.9)
  X_train,y_train=X[:l],y[:l]
  X_test,y_test=X[l:],y[l:]

  log = linear_model.LogisticRegression(C=1e5)
  log.fit(X_train,y_train)

  y_pred= log.predict(X_test)
  errorlist.append(mean_squared_error(y_test,y_pred, squared=False))
  r2scorelist.append(r2_score(y_test,y_pred))

print(np.average(errorlist),np.average(r2scorelist))

from sklearn import svm
import pandas as pd
import numpy as np
df5=pd.read_csv('data - data.csv')
errorlist=[]
r2scorelist=[]
for k in range (1):
  df5=df5.sample(frac=1)
  X, y = df5[["neighbourhood","latitude","longitude","number_of_reviews","commonlabels"]],df5["price"]
  X.commonlabels=pd.Categorical(X.commonlabels)
  X.commonlabels=X.commonlabels.cat.codes
  X.neighbourhood=pd.Categorical(X.neighbourhood)
  X.neighbourhood=X.neighbourhood.cat.codes
  l=int(len(X)*0.9)
  X_train,y_train=X[:l],y[:l]
  X_test,y_test=X[l:],y[l:]

  svc=svm.SVC(kernel='linear')
  svc.fit(X_train, y_train)

  y_pred= svc.predict(X_test)
  errorlist.append(mean_squared_error(y_test,y_pred, squared=False))
  r2scorelist.append(r2_score(y_test,y_pred))

print(np.average(errorlist),np.average(r2scorelist))

y_pred=[np.mean(y_train) for y in y_test]
from sklearn.metrics import mean_absolute_error,mean_squared_error,r2_score
print(mean_absolute_error(y_test,y_pred))
print(mean_squared_error(y_test,y_pred,squared=False))
print(r2_score(y_test,y_pred))

"""# Google Erath Engine - Big Earth Net"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import requests
!pip install openrouteservice
import openrouteservice as opn
from openrouteservice.directions import directions
from openrouteservice import convert
!pip install folium
!pip install geemap==0.5
import folium
#import geemap
import geemap.eefolium as geemap
import ee

# Trigger the authentication flow.
ee.Authenticate()

# Initialize the library.
ee.Initialize()

from pprint import pprint
# Load watersheds from a Fusion Table.
region=ee.Geometry.Polygon(
    [[
      [-9.97708574059, 51.6693012559],
      [-9.97708574059, 55.1316222195],
      [-6.03298539878, 55.1316222195],
      [-6.03298539878, 51.6693012559]
    ]]);
sentinel2 = ee.FeatureCollection('TUBerlin/BigEarthNet/v1').filterBounds(region)#.sort('system:time_start', False)
print(sentinel2.flatten())

s=sentinel2.limit(10).getInfo()['features']
cords=[]
labels=[]
for si in s:
  labels.append(si['properties']['labels'])
  cords.append(si['properties']['system:footprint']['coordinates'])
c=np.array(cords)
print(cords)
#pprint(c)
#for k in c:
#  plt.plot(k[:,0],k[:,1])
#plt.show()
be_labels=pd.DataFrame(columns=["cords","labels"])
be_labels.cords=cords
be_labels.labels=labels
be_labels

from google.colab import drive
drive.mount('drive')

import time
task = ee.batch.Export.table.toDrive(
    collection= sentinel2,
    description='geo',
    folder='Example_folder',
    fileFormat= 'GeoJSON')
task.start()
while task.active():
  print('Status: {}.'.format(task.status))
  time.sleep(5)

be_geo=gpd.read_file("/content/drive/My Drive/Example_folder/geo.geojson")
be_geo

be_geo.plot(color='blue')

m = folium.Map([np.mean(listings.latitude),np.mean(listings.longitude)], zoom_start=7, tiles='cartodbpositron')
folium.GeoJson(be_geo[:2000]).add_to(m)
folium.GeoJson(locs_gdf[7000:10000]).add_to(m)
m

print(be_geo["system:footprint"][0])