From 4141aab1bbbcf55a55e7479e410a873cea64aa03 Mon Sep 17 00:00:00 2001 From: Rami Date: Thu, 4 Jan 2024 20:53:19 +0100 Subject: [PATCH] countries --- core/admin/__init__.py | 1 + core/admin/countries.py | 10 +++ core/management/commands/scrap_countries.py | 13 ++++ core/migrations/0024_country.py | 31 ++++++++++ core/migrations/0025_alter_country_code.py | 17 ++++++ core/models/__init__.py | 3 +- core/models/countries.py | 67 +++++++++++++++++++++ core/models/cvs.py | 2 - requirements.in | 3 +- requirements.txt | 4 +- 10 files changed, 146 insertions(+), 5 deletions(-) create mode 100644 core/admin/countries.py create mode 100644 core/management/commands/scrap_countries.py create mode 100644 core/migrations/0024_country.py create mode 100644 core/migrations/0025_alter_country_code.py create mode 100644 core/models/countries.py diff --git a/core/admin/__init__.py b/core/admin/__init__.py index 75e6c34b..4b14a41a 100644 --- a/core/admin/__init__.py +++ b/core/admin/__init__.py @@ -4,3 +4,4 @@ from .profiles import * # noqa from .tex import * # noqa from .users import * # noqa +from .countries import * # noqa diff --git a/core/admin/countries.py b/core/admin/countries.py new file mode 100644 index 00000000..cad3aa13 --- /dev/null +++ b/core/admin/countries.py @@ -0,0 +1,10 @@ +from django.contrib import admin + +from ..models.countries import Country + + +@admin.register(Country) +class CountryAdmin(admin.ModelAdmin): + list_display = ("name", "code", "gdp", "currency", "wikipedia_url") + list_filter = ("currency",) + readonly_fields = list_display diff --git a/core/management/commands/scrap_countries.py b/core/management/commands/scrap_countries.py new file mode 100644 index 00000000..e04fa885 --- /dev/null +++ b/core/management/commands/scrap_countries.py @@ -0,0 +1,13 @@ +from django.core.management.base import BaseCommand + + +from core.models.countries import Country + + +class Command(BaseCommand): + help = "Scrap object from wikipedia and save in the db" + + def handle(self, *args, **options): + self.stdout.write("Scrapping and saving...") + Country.scrap_from_wikipedia() + self.stdout.write("Done!") diff --git a/core/migrations/0024_country.py b/core/migrations/0024_country.py new file mode 100644 index 00000000..18837d4d --- /dev/null +++ b/core/migrations/0024_country.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.5 on 2024-01-04 19:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0023_user_asked_to_verify_email"), + ] + + operations = [ + migrations.CreateModel( + name="Country", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("code", models.CharField(max_length=8)), + ("name", models.CharField(max_length=64)), + ("gdp", models.PositiveIntegerField(default=50000)), + ("currency", models.CharField(max_length=8)), + ("wikipedia_url", models.URLField(max_length=128)), + ], + ), + ] diff --git a/core/migrations/0025_alter_country_code.py b/core/migrations/0025_alter_country_code.py new file mode 100644 index 00000000..f1c9f11b --- /dev/null +++ b/core/migrations/0025_alter_country_code.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.5 on 2024-01-04 19:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0024_country"), + ] + + operations = [ + migrations.AlterField( + model_name="country", + name="code", + field=models.CharField(db_index=True, max_length=8, unique=True), + ), + ] diff --git a/core/models/__init__.py b/core/models/__init__.py index 7c7d4e2c..241ff3cc 100644 --- a/core/models/__init__.py +++ b/core/models/__init__.py @@ -5,4 +5,5 @@ from .profiles import Profile # noqa from .tex import Tex # noqa from .users import User # noqa -from .users import UserPremiumPlan # noqa1 +from .users import UserPremiumPlan # noqa +from .countries import Country # noqa diff --git a/core/models/countries.py b/core/models/countries.py new file mode 100644 index 00000000..6dbaeb95 --- /dev/null +++ b/core/models/countries.py @@ -0,0 +1,67 @@ +import requests +from bs4 import BeautifulSoup + +from django.db import models + + +class Country(models.Model): + code = models.CharField(max_length=8, unique=True, db_index=True) + name = models.CharField(max_length=64) + gdp = models.PositiveIntegerField(default=50_000) + currency = models.CharField(max_length=8) + wikipedia_url = models.URLField(max_length=128) + + def __str__(self): + return self.code + " " + self.name + + @classmethod + def scrap_from_wikipedia(cls, delete_objs=False): + if delete_objs: + cls.objects.all().delete() + if cls.objects.count() > 0: + return + host = "https://en.wikipedia.org" + url = host + "/wiki/List_of_countries_by_GDP_(nominal)_per_capita" + r = requests.get(url) + soup = BeautifulSoup(r.text, "html.parser") + table = soup.find("table", {"class": "wikitable"}) + for row in table.tbody.find_all("tr"): + name, gdp, c_url = None, None, None + columns = row.find_all("td") + if columns != []: + name = columns[0].text.strip() + c_url = host + columns[0].find_next("a").get("href") + gdp = columns[2].text.strip().replace(",", "") + c_r = requests.get(c_url) + c_soup = BeautifulSoup(c_r.text, "html.parser") + c_table = c_soup.find("table", {"class": "vcard"}) + print(c_url) + currency, code = None, None + for c_tr in c_table.tbody.find_all("tr"): + for c_child in c_tr.children: + if c_child.name == "th": + td = c_child.next_sibling + if c_child.text == "Currency": + try: + anchors = td.find_all("a") + currency = [ + a + for a in anchors + if "ISO 4217" in a.get("title") + ][0].text + except Exception as e: + print(e) + + if "ISO 3166" in c_child.text: + code = td.find_all("a")[0].text + properties = (name, gdp, c_url, currency, code) + if all(properties): + cls.objects.create( + name=name, + code=code, + currency=currency, + wikipedia_url=c_url, + gdp=gdp, + ) + else: + print(f"## Error {properties} ") diff --git a/core/models/cvs.py b/core/models/cvs.py index fd9465b5..3771e535 100644 --- a/core/models/cvs.py +++ b/core/models/cvs.py @@ -1,6 +1,5 @@ import tempfile import time -from functools import cache from pathlib import Path from subprocess import CalledProcessError from subprocess import PIPE @@ -17,7 +16,6 @@ from django.utils import timezone from ..exceptions import TexError -from .profiles import Profile now = timezone.now() diff --git a/requirements.in b/requirements.in index 9e9b0e34..b33f777b 100644 --- a/requirements.in +++ b/requirements.in @@ -50,4 +50,5 @@ black pylint-django async-timeout huey -geoip2 \ No newline at end of file +geoip2 +beautifulsoup4 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 17079a5f..e320b3aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,9 @@ babel==2.12.1 backcall==0.2.0 # via ipython beautifulsoup4==4.11.2 - # via wagtail + # via + # -r requirements.in + # wagtail black==23.9.1 # via -r requirements.in boto3==1.28.53