Merge branch 'soxoj-main'

overcuriousity · Dec 14, 2024 · 5b7d8de · 5b7d8de
2 parents dac9abe + 1e74b09
commit 5b7d8de
Show file tree

Hide file tree

Showing 11 changed files with 248 additions and 212 deletions.
diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml
@@ -1,28 +1,30 @@
-name: Upload Python Package
+name: Upload Python Package to PyPI when a Release is Created
 
 on:
   release:
     types: [created]
 
 jobs:
-  deploy:
-
+  pypi-publish:
+    name: Publish release to PyPI
     runs-on: ubuntu-latest
-
+    environment:
+      name: pypi
+      url: https://pypi.org/p/maigret
+    permissions:
+      id-token: write
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.x'
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine
-    - name: Build and publish
-      env:
-        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-      run: |
-        python setup.py sdist bdist_wheel
-        twine upload dist/*
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel
+      - name: Build package
+        run: |
+          python setup.py sdist bdist_wheel  # Could also be python -m build
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -6,8 +6,8 @@
 copyright = '2024, soxoj'
 author = 'soxoj'
 
-release = '0.4.4'
-version = '0.4.4'
+release = '0.5.0a1'
+version = '0.5'
 
 # -- General configuration
 

diff --git a/maigret/__version__.py b/maigret/__version__.py
@@ -1,3 +1,3 @@
 """Maigret version file"""
 
-__version__ = '0.4.4'
+__version__ = '0.5.0a1'
diff --git a/maigret/sites.py b/maigret/sites.py
@@ -564,8 +564,10 @@ def get_db_stats(self, is_markdown=False):
         site_with_probing = []
         site_with_activation = []
         for site in sites_dict.values():
+
             def get_site_label(site):
                 return f"{site.name}{' (disabled)' if site.disabled else ''}"
+
             if site.url_probe:
                 site_with_probing.append(get_site_label(site))
             if site.activation:

diff --git a/maigret/submit.py b/maigret/submit.py
@@ -453,7 +453,9 @@ async def dialog(self, url_exists, cookie_file):
 
         # Check if the site check is ordinary or not
         if old_site and (old_site.url_probe or old_site.activation):
-            skip = input(f"{Fore.RED}[!] The site check depends on activation / probing mechanism! Consider to update it manually. Continue? [yN]{Style.RESET_ALL}")
+            skip = input(
+                f"{Fore.RED}[!] The site check depends on activation / probing mechanism! Consider to update it manually. Continue? [yN]{Style.RESET_ALL}"
+            )
             if skip.lower() in ['n', '']:
                 return False
 
@@ -604,7 +606,9 @@ async def dialog(self, url_exists, cookie_file):
         if old_site:
             default_tags_str = f' [{", ".join(old_site.tags)}]'
 
-        new_tags = input(f"{Fore.GREEN}[?] Site tags{default_tags_str}: {Style.RESET_ALL}")
+        new_tags = input(
+            f"{Fore.GREEN}[?] Site tags{default_tags_str}: {Style.RESET_ALL}"
+        )
         if new_tags:
             chosen_site.tags = list(map(str.strip, new_tags.split(',')))
         else:

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "maigret"
-version = "0.4.4"
+version = "0.5.0a1"
 description = "🕵️‍♂️ Collect a dossier on a person by username from thousands of sites."
 authors = ["Soxoj <[email protected]>"]
 readme = "README.md"
@@ -72,22 +72,23 @@ reportlab = "^4.2.0"
 cloudscraper = "^1.2.71"
 flask = {extras = ["async"], version = "^3.1.0"}
 asgiref = "^3.8.1"
+platformdirs = "^4.3.6"
 
 
 [tool.poetry.group.dev.dependencies]
 # How to add a new dev dependency: poetry add black --group dev
 # Install dev dependencies with: poetry install --with dev
 flake8 = "^7.1.1"
 pytest = "^8.3.4"
-pytest-asyncio = "^0.24.0"
+pytest-asyncio = "^0.25.0"
 pytest-cov = "^6.0.0"
 pytest-httpserver = "^1.0.0"
 pytest-rerunfailures = "^15.0"
 reportlab = "^4.2.0"
 mypy = "^1.13.0"
 tuna = "^0.5.11"
-black = "^24.10.0"
 coverage = "^7.6.9"
+black = "^24.10.0"
 
 [tool.poetry.scripts]
 # Run with: poetry run maigret <username>

diff --git a/sites.md b/sites.md
@@ -3141,7 +3141,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://pubg.op.gg) [OP.GG [PUBG] (https://pubg.op.gg)](https://pubg.op.gg)*: top 100M, gaming*
 1. ![](https://www.google.com/s2/favicons?domain=https://valorant.op.gg) [OP.GG [Valorant] (https://valorant.op.gg)](https://valorant.op.gg)*: top 100M, gaming*
 
-The list was updated at (2024-12-12)
+The list was updated at (2024-12-13)
 ## Statistics
 
 Enabled/total sites: 2684/3137 = 85.56%

diff --git a/snapcraft.yaml b/snapcraft.yaml
@@ -7,7 +7,7 @@ description: |
   
   Currently supported more than 3000 sites, search is launched against 500 popular sites in descending order of popularity by default. Also supported checking of Tor sites, I2P sites, and domains (via DNS resolving).
 
-version: 0.4.4
+version: 0.5.0a1
 license: MIT
 base: core22
 confinement: strict

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -8,8 +8,11 @@
 from maigret.sites import MaigretDatabase
 from maigret.maigret import setup_arguments_parser
 from maigret.settings import Settings
+from aiohttp import web
 
 
+LOCAL_SERVER_PORT = 8080
+
 CUR_PATH = os.path.dirname(os.path.realpath(__file__))
 JSON_FILE = os.path.join(CUR_PATH, '../maigret/resources/data.json')
 SETTINGS_FILE = os.path.join(CUR_PATH, '../maigret/resources/settings.json')
@@ -96,3 +99,20 @@ def argparser():
 @pytest.fixture(scope="session")
 def httpserver_listen_address():
     return ("localhost", 8989)
+
+
+@pytest.fixture
+async def cookie_test_server():
+    async def handle_cookies(request):
+        print(f"Received cookies: {request.cookies}")
+        cookies_dict = {k: v for k, v in request.cookies.items()}
+        return web.json_response({'cookies': cookies_dict})
+
+    app = web.Application()
+    app.router.add_get('/cookies', handle_cookies)
+    runner = web.AppRunner(app)
+    await runner.setup()
+    server = web.TCPSite(runner, port=LOCAL_SERVER_PORT)
+    await server.start()
+    yield server
+    await runner.cleanup()
diff --git a/tests/test_activation.py b/tests/test_activation.py
@@ -1,11 +1,13 @@
 """Maigret activation test functions"""
 
 import json
+import yarl
 
 import aiohttp
 import pytest
 from mock import Mock
 
+from tests.conftest import LOCAL_SERVER_PORT
 from maigret.activation import ParsingActivator, import_aiohttp_cookies
 
 COOKIES_TXT = """# HTTP Cookie File downloaded with cookies.txt by Genuinous @genuinous
@@ -19,10 +21,11 @@
 xss.is	FALSE	/	TRUE	1642709308	xf_user	tset
 .xss.is	TRUE	/	FALSE	0	muchacho_cache	test
 .xss.is	TRUE	/	FALSE	1924905600	132_evc	test
-httpbin.org	FALSE	/	FALSE	0	a	b
+localhost	FALSE	/	FALSE	0	a	b
 """
 
 
+@pytest.mark.skip("captcha")
 @pytest.mark.slow
 def test_vimeo_activation(default_db):
     vimeo_site = default_db.sites_dict['Vimeo']
@@ -36,27 +39,20 @@ def test_vimeo_activation(default_db):
 
 @pytest.mark.slow
 @pytest.mark.asyncio
-async def test_import_aiohttp_cookies():
+async def test_import_aiohttp_cookies(cookie_test_server):
     cookies_filename = 'cookies_test.txt'
     with open(cookies_filename, 'w') as f:
         f.write(COOKIES_TXT)
 
     cookie_jar = import_aiohttp_cookies(cookies_filename)
-    # new aiohttp support
-    assert list(cookie_jar._cookies.keys()) in (
-        ['xss.is', 'httpbin.org'],
-        [('xss.is', '/'), ('httpbin.org', '/')],
-        [('xss.is', ''), ('httpbin.org', '')],
-    )
-
-    url = 'https://httpbin.org/cookies'
-    connector = aiohttp.TCPConnector(ssl=False)
-    session = aiohttp.ClientSession(
-        connector=connector, trust_env=True, cookie_jar=cookie_jar
-    )
-
-    response = await session.get(url=url)
-    result = json.loads(await response.content.read())
-    await session.close()
+    url = f'http://localhost:{LOCAL_SERVER_PORT}/cookies'
+
+    cookies = cookie_jar.filter_cookies(yarl.URL(url))
+    assert cookies['a'].value == 'b'
+
+    async with aiohttp.ClientSession(cookie_jar=cookie_jar) as session:
+        async with session.get(url=url) as response:
+            result = await response.json()
+            print(f"Server response: {result}")
 
     assert result == {'cookies': {'a': 'b'}}