Merge pull request #15 from deedy5/v2.1

v2.1
1)    code redesigned to remove lxml and brotli from the requirements,
2)    dded lru_cache for the function get_vqd(),
3)    time delays removed.

duckduckgo_search/ddg.py

@@ -1,11 +1,8 @@
-import json
 import logging
-from datetime import datetime
-from time import sleep
 
 from requests import ConnectionError
 
-from .utils import _normalize, _save_csv, _save_json, get_vqd, session
+from .utils import _do_output, _get_vqd, _normalize, session
 
 logger = logging.getLogger(__name__)
 
@@ -33,12 +30,11 @@ def ddg(
     """
 
     if not keywords:
-        return
+        return None
 
-    vqd = get_vqd(keywords)
+    vqd = _get_vqd(keywords)
     if not vqd:
-        return
-    sleep(0.75)
+        return None
 
     # search
     safesearch_base = {"On": 1, "Moderate": -1, "Off": -2}
@@ -96,7 +92,6 @@ def ddg(
         if not page_results:
             break
         results.extend(page_results)
-        sleep(0.75)
 
     """ using html method
     payload = {
@@ -126,16 +121,6 @@ def ddg(
         sleep(2)
     """
     results = results[:max_results]
-
-    # output
-    keywords = keywords.replace('"', "'")
-    if output == "csv":
-        _save_csv(f"ddg_{keywords}_{datetime.now():%Y%m%d_%H%M%S}.csv", results)
-    elif output == "json":
-        _save_json(f"ddg_{keywords}_{datetime.now():%Y%m%d_%H%M%S}.json", results)
-    elif output == "print":
-        for i, result in enumerate(results, start=1):
-            print(f"{i}.", json.dumps(result, ensure_ascii=False, indent=4))
-            input()
-
+    if output:
+        _do_output(__name__, keywords, output, results)
     return results

duckduckgo_search/ddg_images.py

@@ -1,13 +1,11 @@
-import json
 import logging
 import os
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime
-from time import sleep
 
 from requests import ConnectionError
 
-from .utils import _download_image, _save_csv, _save_json, get_vqd, session
+from .utils import _do_output, _download_image, _get_vqd, session
 
 logger = logging.getLogger(__name__)
 
@@ -57,10 +55,9 @@ def ddg_images(
     if not keywords:
         return None
 
-    vqd = get_vqd(keywords)
+    vqd = _get_vqd(keywords)
     if not vqd:
-        return
-    sleep(0.75)
+        return None
 
     # get images
     safesearch_base = {"On": 1, "Moderate": -1, "Off": -2}
@@ -119,24 +116,12 @@ def ddg_images(
         results.extend(page_results)
         # pagination
         payload["s"] += 100
-        sleep(0.2)
 
     results = results[:max_results]
+    if output:
+        _do_output(__name__, keywords, output, results)
 
-    # output
-    keywords = keywords.replace('"', "'")
-    if output == "csv":
-        _save_csv(f"ddg_images_{keywords}_{datetime.now():%Y%m%d_%H%M%S}.csv", results)
-    elif output == "json":
-        _save_json(
-            f"ddg_images_{keywords}_{datetime.now():%Y%m%d_%H%M%S}.json", results
-        )
-    elif output == "print":
-        for i, result in enumerate(results, start=1):
-            print(f"{i}.", json.dumps(result, ensure_ascii=False, indent=2))
-            input()
-
-    # download
+    # download images
     if download:
         print("Downloading images. Wait...")
         keywords = keywords.replace('"', "'")
@@ -145,6 +130,7 @@ def ddg_images(
         futures = []
         with ThreadPoolExecutor(30) as executor:
             for i, res in enumerate(results, start=1):
+                print(i)
                 filename = res["image"].split("/")[-1].split("?")[0]
                 future = executor.submit(
                     _download_image, res["image"], path, f"{i}_{filename}"

duckduckgo_search/ddg_maps.py

@@ -1,15 +1,12 @@
-import json
 import logging
 from collections import deque
 from dataclasses import dataclass
-from datetime import datetime
 from decimal import Decimal
-from time import sleep
 
 import requests
 from requests import ConnectionError
 
-from .utils import _normalize, _save_csv, _save_json, get_vqd, headers, session
+from .utils import _do_output, _get_vqd, _normalize, headers, session
 
 logger = logging.getLogger(__name__)
 
@@ -73,10 +70,9 @@ def ddg_maps(
         return None
 
     # get vqd
-    vqd = get_vqd(keywords)
+    vqd = _get_vqd(keywords)
     if not vqd:
-        return
-    sleep(0.75)
+        return None
 
     # if longitude and latitude are specified, skip the request about bbox to the nominatim api
     if latitude and longitude:
@@ -204,22 +200,8 @@ def ddg_maps(
             work_bboxes.extendleft([bbox1, bbox2, bbox3, bbox4])
 
         print(f"Found {len(results)}")
-        sleep(0.2)
-
-    # output
-    keywords = keywords.replace('"', "'")
-    if output == "csv":
-        _save_csv(
-            f"ddg_maps_{keywords}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv",
-            results,
-        )
-    elif output == "json":
-        _save_json(
-            f"ddg_maps_{keywords}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json",
-            results,
-        )
-    elif output == "print":
-        for i, result in enumerate(results, start=1):
-            print(f"{i}.", json.dumps(result, ensure_ascii=False, indent=2))
-            input()
+
+    results = results[:max_results]
+    if output:
+        _do_output(__name__, keywords, output, results)
     return results

duckduckgo_search/ddg_news.py

@@ -1,11 +1,9 @@
-import json
 import logging
 from datetime import datetime
-from time import sleep
 
 from requests import ConnectionError
 
-from .utils import _normalize, _save_csv, _save_json, get_vqd, session
+from .utils import _do_output, _get_vqd, _normalize, session
 
 logger = logging.getLogger(__name__)
 
@@ -36,10 +34,9 @@ def ddg_news(
         return None
 
     # get vqd
-    vqd = get_vqd(keywords)
+    vqd = _get_vqd(keywords)
     if not vqd:
-        return
-    sleep(0.75)
+        return None
 
     # get news
     safesearch_base = {"On": 1, "Moderate": -1, "Off": -2}
@@ -92,26 +89,8 @@ def ddg_news(
         results.extend(page_results)
         # pagination
         params["s"] += 30
-        sleep(0.2)
-
-    results = results[:max_results]
-    # sort by datetime
-    results = sorted(results, key=lambda x: x["date"], reverse=True)
 
-    # output
-    keywords = keywords.replace('"', "'")
-    if output == "csv":
-        _save_csv(
-            f"ddg_news_{keywords}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv",
-            results,
-        )
-    elif output == "json":
-        _save_json(
-            f"ddg_news_{keywords}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json",
-            results,
-        )
-    elif output == "print":
-        for i, result in enumerate(results, start=1):
-            print(f"{i}.", json.dumps(result, ensure_ascii=False, indent=2))
-            input()
+    results = sorted(results[:max_results], key=lambda x: x["date"], reverse=True)
+    if output:
+        _do_output(__name__, keywords, output, results)
     return results

duckduckgo_search/ddg_translate.py

@@ -1,11 +1,8 @@
-import json
 import logging
-from datetime import datetime
-from time import sleep
 
 from requests import ConnectionError
 
-from .utils import _save_csv, _save_json, get_vqd, session
+from .utils import _do_output, _get_vqd, session
 
 logger = logging.getLogger(__name__)
 
@@ -32,10 +29,9 @@ def ddg_translate(
         return None
 
     # get vqd
-    vqd = get_vqd("translate")
+    vqd = _get_vqd("translate")
     if not vqd:
-        return
-    sleep(0.75)
+        return None
 
     # translate
     params = {
@@ -66,22 +62,8 @@ def ddg_translate(
             logger.error("Connection Error.")
         except Exception:
             logger.exception("Exception.", exc_info=True)
-        sleep(0.2)
 
-    # output
-    keywords = keywords[0].replace('"', "'")
-    if output == "csv":
-        _save_csv(
-            f"ddg_translate_{keywords}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv",
-            results,
-        )
-    elif output == "json":
-        _save_json(
-            f"ddg_translate_{keywords}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json",
-            results,
-        )
-    elif output == "print":
-        for i, result in enumerate(results, start=1):
-            print(f"{i}.", json.dumps(result, ensure_ascii=False, indent=2))
-            input()
+    if output:
+        keywords = keywords[0]
+        _do_output(__name__, keywords, output, results)
     return results

duckduckgo_search/ddg_videos.py

@@ -1,11 +1,8 @@
-import json
 import logging
-from datetime import datetime
-from time import sleep
 
 from requests import ConnectionError
 
-from .utils import _save_csv, _save_json, get_vqd, session
+from .utils import _do_output, _get_vqd, session
 
 logger = logging.getLogger(__name__)
 
@@ -39,13 +36,12 @@ def ddg_videos(
     """
 
     if not keywords:
-        return
+        return None
 
     # get vqd
-    vqd = get_vqd(keywords)
+    vqd = _get_vqd(keywords)
     if not vqd:
-        return
-    sleep(0.75)
+        return None
 
     # get videos
     safesearch_base = {"On": 1, "Moderate": -1, "Off": -2}
@@ -93,25 +89,8 @@ def ddg_videos(
         results.extend(page_results)
         # for pagination
         payload["s"] += 62
-        sleep(0.2)
 
     results = results[:max_results]
-
-    # output
-    keywords = keywords.replace('"', "'")
-    if output == "csv":
-        _save_csv(
-            f"ddg_videos_{keywords}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv",
-            results,
-        )
-    elif output == "json":
-        _save_json(
-            f"ddg_videos_{keywords}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json",
-            results,
-        )
-    elif output == "print":
-        for i, result in enumerate(results, start=1):
-            print(f"{i}.", json.dumps(result, ensure_ascii=False, indent=2))
-            input()
-
+    if output:
+        _do_output(__name__, keywords, output, results)
     return results