removed nltk test, restructured files

backend/classifiers/nltktest.py

@@ -14,7 +14,7 @@
 import rootpath
 
 rootpath.append()
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 
 
 class NLTKTest:

backend/data_preparation/connection.py → backend/connection.py

@@ -9,7 +9,7 @@
 
 rootpath.append()
 from paths import DATABASE_CONFIG_PATH
-from utilities.ini_parser import parse
+from backend.utilities.ini_parser import parse
 
 logger = logging.getLogger('TaskManager')
 

backend/data_preparation/crawler/noaa_crawler.py

@@ -1,11 +1,11 @@
 import logging
+import math
 import os
 import sys
 import time
 import traceback
 from datetime import datetime, timedelta, timezone
 
-import math
 import requests
 import rootpath
 
@@ -14,7 +14,7 @@
 rootpath.append()
 
 from paths import GRIB2_DATA_DIR
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 from backend.data_preparation.crawler.crawlerbase import CrawlerBase
 from backend.data_preparation.dumper.noaa_dumper import NOAADumper
 

backend/data_preparation/crawler/soil_mois_crawler.py

@@ -11,7 +11,7 @@
 import wget
 
 rootpath.append()
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 from paths import SOIL_MOIS_DATA_DIR
 from backend.data_preparation.crawler.crawlerbase import CrawlerBase
 

backend/data_preparation/crawler/twitter_crawler.py

@@ -12,13 +12,13 @@
 rootpath.append()
 
 from paths import TWITTER_API_CONFIG_PATH
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 from backend.data_preparation.crawler.crawlerbase import CrawlerBase
 from backend.data_preparation.dumper.twitter_dumper import TweetDumper
 from backend.data_preparation.extractor.twitter_extractor import TweetExtractor
 from backend.data_preparation.dumper.dumperbase import DumperBase, DumperException
 from backend.data_preparation.extractor.extractorbase import ExtractorBase, ExtractorException
-from utilities.ini_parser import parse
+from backend.utilities.ini_parser import parse
 
 logger = logging.getLogger('TaskManager')
 

backend/data_preparation/dumper.py

@@ -4,7 +4,7 @@
 import rootpath
 
 rootpath.append()
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 
 
 class Dumper:

backend/data_preparation/dumper/event2mind_dumper.py

@@ -2,14 +2,14 @@
 @author: Yutong Wang
 """
 import logging
-import psycopg2
 import traceback
 
+import psycopg2
 import rootpath
 from psycopg2 import extras
 
 rootpath.append()
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 from backend.data_preparation.dumper.dumperbase import DumperBase
 
 logger = logging.getLogger('TastManager')

backend/data_preparation/dumper/img_classification_dumper.py

@@ -10,7 +10,7 @@
 rootpath.append()
 from backend.data_preparation.dumper.dumperbase import DumperBase
 from backend.classifiers.image_classifier import ImageClassifier
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 
 logger = logging.getLogger('TaskManager')
 

backend/data_preparation/dumper/noaa_dumper.py

@@ -8,7 +8,7 @@
 import rootpath
 
 rootpath.append()
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 from backend.data_preparation.dumper.dumperbase import DumperBase
 
 logger = logging.getLogger('TaskManager')

backend/data_preparation/dumper/prism_dumper.py

@@ -9,7 +9,7 @@
 import rootpath
 
 rootpath.append()
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 from backend.data_preparation.dumper.dumperbase import DumperBase
 from backend.data_preparation.crawler.usgs_crawler import USGSCrawler
 from backend.data_preparation.extractor.soil_mois_extractor import TiffExtractor

backend/data_preparation/dumper/soil_mois_dumper.py

@@ -13,7 +13,7 @@
 from backend.data_preparation.extractor.soil_mois_extractor import TiffExtractor
 
 from backend.data_preparation.dumper.dumperbase import DumperBase
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 
 logger = logging.getLogger('TaskManager')
 

backend/data_preparation/dumper/text_classification_dumper.py

@@ -2,7 +2,7 @@
 
 rootpath.append()
 from backend.data_preparation.dumper.dumperbase import DumperBase
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 
 
 class TextClassificationDumper(DumperBase):

backend/data_preparation/dumper/twitter_dumper.py

@@ -8,7 +8,7 @@
 
 rootpath.append()
 
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 
 from backend.data_preparation.dumper.dumperbase import DumperBase
 

backend/data_preparation/dumper/url_dumper.py

@@ -4,7 +4,7 @@
 import rootpath
 
 rootpath.append()
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 from backend.data_preparation.dumper.dumperbase import DumperBase
 
 logger = logging.getLogger('TaskManager')

backend/data_preparation/labeler.py

@@ -4,7 +4,7 @@
 import rootpath
 
 rootpath.append()
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 
 
 class Labeler:

backend/task/data_from_prism.py

@@ -8,7 +8,7 @@
 
 rootpath.append()
 from backend.task.runnable import Runnable
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 from backend.data_preparation.crawler.prism_crawler import PRISMCrawler
 from backend.data_preparation.extractor.bil_extractor import BILExtractor, BILFormat
 from backend.data_preparation.dumper.prism_dumper import PRISMDumper

backend/task/data_from_usgs.py

@@ -13,7 +13,7 @@
 from backend.data_preparation.crawler.usgs_crawler import USGSCrawler
 from backend.data_preparation.extractor.soil_mois_extractor import TiffExtractor
 from backend.data_preparation.dumper.prism_dumper import PRISMDumper
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 
 logger = logging.getLogger('TaskManager')
 

backend/task/event2mind_classification_runnable.py

@@ -7,7 +7,7 @@
 import rootpath
 
 rootpath.append()
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 from backend.task.runnable import Runnable
 from backend.classifiers.event2mind_classifier import Event2MindClassifier
 from backend.data_preparation.dumper.event2mind_dumper import Event2MindDumper

backend/task/image_classification_runnable.py

@@ -8,7 +8,7 @@
 
 rootpath.append()
 from backend.task.runnable import Runnable
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 from backend.classifiers.image_classifier import ImageClassifier
 from backend.data_preparation.dumper.img_classification_dumper import ImgClassificationDumper
 

backend/task/image_from_tweet.py

@@ -4,7 +4,7 @@
 import rootpath
 
 rootpath.append()
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 from backend.data_preparation.dumper.url_dumper import URLDumper
 from backend.data_preparation.extractor.tweet_media_extractor import TweetMediaExtractor
 from backend.task.runnable import Runnable

backend/task/text_classification_runnable.py

@@ -5,7 +5,7 @@
 
 rootpath.append()
 from backend.task.runnable import Runnable
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 from backend.classifiers.text_classifier import TextClassifier
 from backend.data_preparation.dumper.text_classification_dumper import TextClassificationDumper
 from paths import TEXT_CNN_MODEL_PATH

backend/web/__init__.py

@@ -8,10 +8,7 @@
 import router.tweet_router
 
 rootpath.append()
-# TODO: change model loading
-from backend.classifiers.nltktest import NLTKTest  # must import NLTKTest before root_router
 
-nl: NLTKTest  # to avoid NLTKTest importation being optimized by IDE
 import router.root_router
 
 

backend/web/router/data_router.py

@@ -14,7 +14,7 @@
 from flask import Blueprint, make_response, jsonify, send_from_directory, request as flask_request
 
 rootpath.append()
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 from paths import BOUNDARY_PATH
 
 bp = Blueprint('data', __name__, url_prefix='/data')

backend/web/router/dropDownMenu_router.py

@@ -3,7 +3,7 @@
 from flask import Blueprint, make_response, jsonify, request as flask_request
 
 rootpath.append()
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 
 bp = Blueprint('dropdownMenu', __name__, url_prefix='/dropdownMenu')
 

backend/web/router/root_router.py

@@ -1,16 +1,12 @@
-import pickle
 import time
 
 import rootpath
 from flask import Blueprint, make_response, jsonify, request as flask_request
 
 rootpath.append()
-from backend.data_preparation.connection import Connection
-from paths import NLTK_MODEL_PATH
-from backend.classifiers.nltktest import NLTKTest
+from backend.connection import Connection
 
 bp = Blueprint('root', __name__, url_prefix='/')
-nl: NLTKTest = pickle.load(open(NLTK_MODEL_PATH, 'rb'))
 
 
 @bp.route("/wildfire-prediction", methods=['POST'])
@@ -31,9 +27,11 @@ def send_wildfire():
     end = time.mktime(time.strptime(end, "%Y-%m-%dT%H:%M:%S.%fZ"))
 
     resp = make_response(
-        jsonify([{"long": lon, "lat": lat, "nlp": nl.predict(nlp_text), "text": text} for lon, lat, nlp_text, text in
+        jsonify([{"long": lon, "lat": lat, "nlp": True, "text": text} for lon, lat, nlp_text, text in
                  Connection().sql_execute(
                      f"select l.top_left_long, l.top_left_lat, r.text, r.text from locations l, images i, records r "
-                     f"where l.id = i.id and r.id = l.id and i.wildfire_prob>0.9 and l.top_left_long>{west} and l.top_left_lat<{north} "
-                     f"and l.bottom_right_long<{east} and l.bottom_right_lat>{south} and extract(epoch from r.create_at) >{start} and extract(epoch from r.create_at) <{end}")]))
+                     f"where l.id = i.id and r.id = l.id and i.wildfire_prob>0.9 and "
+                     f"l.top_left_long>{west} and l.top_left_lat<{north} "
+                     f"and l.bottom_right_long<{east} and l.bottom_right_lat>{south} "
+                     f"and extract(epoch from r.create_at) >{start} and extract(epoch from r.create_at) <{end}")]))
     return resp

backend/web/router/search_router.py

@@ -5,7 +5,7 @@
 from flask import Blueprint, make_response, jsonify, request as flask_request
 
 rootpath.append()
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 
 bp = Blueprint('search', __name__, url_prefix='/search')
 

backend/web/router/tweet_router.py

@@ -9,12 +9,13 @@
 import rootpath
 import twitter
 from flask import Blueprint, make_response, jsonify, request as flask_request
+
 from router.data_router import fill_series, gen_date_series
 
 rootpath.append()
-from backend.data_preparation.connection import Connection
+from backend.connection import Connection
 from paths import TWITTER_API_CONFIG_PATH
-from utilities.ini_parser import parse
+from backend.utilities.ini_parser import parse
 
 bp = Blueprint('tweet', __name__, url_prefix='/tweet')
 api = twitter.Api(**parse(TWITTER_API_CONFIG_PATH, 'twitter-API'))