diff --git a/contributors/management/commands/fetchdata.py b/contributors/management/commands/fetchdata.py index 081e2aa..90dcb22 100644 --- a/contributors/management/commands/fetchdata.py +++ b/contributors/management/commands/fetchdata.py @@ -1,10 +1,9 @@ import logging import sys -from django.db import transaction - import requests from django.core import management +from django.db import transaction from django.utils import dateparse from contributors.models import ( @@ -21,7 +20,7 @@ from contributors.utils import misc # Simultaneous logging to file and stdout -logger = logging.getLogger('GitHub') +logger = logging.getLogger("GitHub") logger.setLevel(logging.INFO) logger.addHandler(logging.StreamHandler(sys.stdout)) @@ -37,7 +36,9 @@ @transaction.atomic -def create_contributions(repo, contrib_data, user_field='user', id_field='id', type_='iss'): # noqa: C901 +def create_contributions( + repo, contrib_data, user_field="user", id_field="id", type_="iss" +): # noqa: C901 """ Создает записи о контрибуциях в базе данных. """ @@ -51,21 +52,27 @@ def create_contributions(repo, contrib_data, user_field='user', id_field='id', t logger.info(contrib) contributor = None if _is_valid_contributor(contrib, user_field): - login = contrib[user_field]['login'] + login = contrib[user_field]["login"] if login not in contributors: - contributor, created = Contributor.objects.get_or_create(login=login) + contributor, _ = Contributor.objects.get_or_create(login=login) contributors[login] = contributor else: contributor = contributors[login] if contributor: - contribution = _create_contribution(repo, contrib, contributor, id_field, type_) + contribution = _create_contribution( + repo, contrib, contributor, id_field, type_ + ) contributions.append(contribution) - if type_ == 'cit': - commit_stats.append(_create_commit_stats(repo, contribution, contrib, id_field)) - elif type_ in {'pr', 'iss'}: - issue_infos.append(_create_issue_info(repo, contribution, contrib, type_)) + if type_ == "cit": + commit_stats.append( + _create_commit_stats(repo, contribution, contrib, id_field) + ) + elif type_ in {"pr", "iss"}: + issue_infos.append( + _create_issue_info(repo, contribution, contrib, type_) + ) contribution_labels.extend(_create_contribution_labels(contrib)) _bulk_create_records(contributions, commit_stats, issue_infos, contribution_labels) @@ -75,52 +82,65 @@ def create_contributions(repo, contrib_data, user_field='user', id_field='id', t def _is_valid_contributor(contrib, user_field): """Проверяет, является ли контрибуция допустимой для обработки.""" contrib_author = contrib[user_field] - return (contrib_author is not None - and contrib_author['type'] != 'Bot' - and contrib_author['login'] not in IGNORED_CONTRIBUTORS) + return ( + contrib_author is not None + and contrib_author["type"] != "Bot" + and contrib_author["login"] not in IGNORED_CONTRIBUTORS + ) def _create_contribution(repo, contrib, contributor, id_field, type_): """Создает объект Contribution.""" - true_type = 'pr' if (type_ == 'iss' and 'pull_request' in contrib) else type_ - datetime = contrib['commit']['author']['date'] if true_type == 'cit' else contrib['created_at'] + true_type = "pr" if (type_ == "iss" and "pull_request" in contrib) else type_ + datetime = ( + contrib["commit"]["author"]["date"] + if true_type == "cit" + else contrib["created_at"] + ) return Contribution( id=contrib[id_field], repository=repo, contributor=contributor, type=true_type, - html_url=contrib['html_url'], + html_url=contrib["html_url"], created_at=dateparse.parse_datetime(datetime), ) def _create_commit_stats(repo, contribution, contrib, id_field): """Создает объект CommitStats для коммита.""" - commit_data = github.get_commit_data(repo.organization or repo.owner, repo, contrib[id_field], session) + commit_data = github.get_commit_data( + repo.organization or repo.owner, repo, contrib[id_field], session + ) return CommitStats( commit=contribution, - additions=commit_data['stats']['additions'], - deletions=commit_data['stats']['deletions'], + additions=commit_data["stats"]["additions"], + deletions=commit_data["stats"]["deletions"], ) def _create_issue_info(repo, contribution, contrib, type_): """Создает объект IssueInfo для issue или pull request.""" - state = 'merged' if type_ == 'pr' and github.is_pr_merged( - repo.organization or repo.owner, repo, contrib['number'], session - ) else contrib['state'] + state = ( + "merged" + if type_ == "pr" + and github.is_pr_merged( + repo.organization or repo.owner, repo, contrib["number"], session + ) + else contrib["state"] + ) return IssueInfo( issue=contribution, - title=contrib['title'], + title=contrib["title"], state=state, ) def _create_contribution_labels(contrib): """Создает объекты ContributionLabel для вклада.""" - return [ContributionLabel(name=label["name"]) for label in contrib['labels']] + return [ContributionLabel(name=label["name"]) for label in contrib["labels"]] def _bulk_create_records(contributions, commit_stats, issue_infos, contribution_labels): @@ -134,9 +154,13 @@ def _bulk_create_records(contributions, commit_stats, issue_infos, contribution_ @transaction.atomic def _link_labels_to_contributions(contributions, contrib_data): """Связывает метки с контрибуциями.""" - all_label_names = set(label['name'] for c in contrib_data for label in c['labels']) - existing_labels = {label.name: label for label in Label.objects.filter(name__in=all_label_names)} - new_labels = [Label(name=name) for name in all_label_names if name not in existing_labels] + all_label_names = set(label["name"] for c in contrib_data for label in c["labels"]) + existing_labels = { + label.name: label for label in Label.objects.filter(name__in=all_label_names) + } + new_labels = [ + Label(name=name) for name in all_label_names if name not in existing_labels + ] Label.objects.bulk_create(new_labels, ignore_conflicts=True) all_labels = existing_labels.copy() @@ -145,11 +169,16 @@ def _link_labels_to_contributions(contributions, contrib_data): ContributionLabel = Contribution.labels.through contribution_labels = [] - for contribution, labels in zip(contributions, [c['labels'] for c in contrib_data]): - contribution_labels.extend([ - ContributionLabel(contribution_id=contribution.id, label_id=all_labels[label['name']].id) - for label in labels - ]) + for contribution, labels in zip(contributions, [c["labels"] for c in contrib_data]): + contribution_labels.extend( + [ + ContributionLabel( + contribution_id=contribution.id, + label_id=all_labels[label["name"]].id, + ) + for label in labels + ] + ) # Удаляем существующие связи ContributionLabel.objects.filter(contribution__in=contributions).delete() @@ -166,13 +195,15 @@ class Command(management.base.BaseCommand): def add_arguments(self, parser): """Добавляет аргументы для команды.""" parser.add_argument( - 'owner', - nargs='*', + "owner", + nargs="*", default=ORGANIZATIONS, - help='список имен владельцев', + help="список имен владельцев", ) parser.add_argument( - '--repo', nargs='*', help='список полных имен репозиториев', + "--repo", + nargs="*", + help="список полных имен репозиториев", ) def handle(self, *args, **options): @@ -187,19 +218,21 @@ def handle(self, *args, **options): finally: session.close() - logger.info(self.style.SUCCESS( - "Данные получены из GitHub и сохранены в базу данных", - )) + logger.info( + self.style.SUCCESS( + "Данные получены из GitHub и сохранены в базу данных", + ) + ) def _get_data_of_owners_and_repos(self, options): """Получает данные о владельцах и репозиториях.""" - if options['repo']: + if options["repo"]: return github.get_data_of_owners_and_repos( - repo_full_names=options['repo'], + repo_full_names=options["repo"], ) - elif options['owner']: + elif options["owner"]: return github.get_data_of_owners_and_repos( - owner_names=options['owner'], + owner_names=options["owner"], ) else: raise management.base.CommandError( @@ -217,15 +250,15 @@ def _process_owners_and_repos(self, data_of_owners_and_repos): def _process_owner(self, owner_data): """Обрабатывает данные о владельце.""" - owner_type = owner_data['details']['type'] - table = Contributor if owner_type == 'User' else Organization - owner, _ = misc.update_or_create_record(table, owner_data['details']) + owner_type = owner_data["details"]["type"] + table = Contributor if owner_type == "User" else Organization + owner, _ = misc.update_or_create_record(table, owner_data["details"]) logger.info(f"Обработка {owner_type}: {owner}") return owner def _process_organization_repos(self, organization, org_data): """Обрабатывает репозитории организации.""" - repos_data = org_data.get('repos', []) + repos_data = org_data.get("repos", []) number_of_repos = len(repos_data) for i, repo_data in enumerate(repos_data, start=1): self._process_repo(organization, repo_data, i, number_of_repos) @@ -239,15 +272,18 @@ def _process_user_repos(self, user): def _get_repos_to_process(self, owner): """Получает список репозиториев для обработки.""" - return Repository.objects.filter(owner=owner).exclude( - name__in=IGNORED_REPOSITORIES - ).select_related('owner').prefetch_related('labels') + return ( + Repository.objects.filter(owner=owner) + .exclude(name__in=IGNORED_REPOSITORIES) + .select_related("owner") + .prefetch_related("labels") + ) def _process_repo(self, owner, repo_data, i, number_of_repos): """Обрабатывает отдельный репозиторий.""" repo, _ = misc.update_or_create_record(Repository, repo_data) logger.info(f"Обработка репозитория: {repo} ({i}/{number_of_repos})") - if repo_data['size'] == 0: + if repo_data["size"] == 0: logger.info("Пустой репозиторий, пропускаем") return @@ -256,14 +292,14 @@ def _process_repo(self, owner, repo_data, i, number_of_repos): def _process_repo_language(self, repo, repo_data): """Обрабатывает язык репозитория.""" - language = repo_data['language'] + language = repo_data["language"] if language: label, _ = Label.objects.get_or_create(name=language) repo.labels.add(label) def _process_repo_contributions(self, owner, repo): """Обрабатывает вклады в репозиторий.""" - extra_info = {'owner': owner, 'repo': repo} + extra_info = {"owner": owner, "repo": repo} self._process_issues_and_prs(owner, repo, extra_info) self._process_commits(owner, repo, extra_info) self._process_comments(owner, repo, extra_info) @@ -276,16 +312,16 @@ def _process_issues_and_prs(self, owner, repo, extra_info): logger.info(contrib_data) except Exception as e: logger.error( - extra=extra_info | {'ex': str(e)}, + extra=extra_info | {"ex": str(e)}, msg="Не удалось обработать issues и pull requests", ) return create_contributions( repo, contrib_data, - user_field='user', - id_field='id', - type_='iss', + user_field="user", + id_field="id", + type_="iss", ) def _process_commits(self, owner, repo, extra_info): @@ -293,20 +329,22 @@ def _process_commits(self, owner, repo, extra_info): logger.info("Обработка коммитов") try: contrib_data = github.get_repo_commits_except_merges( - owner, repo, session=session, + owner, + repo, + session=session, ) except Exception as e: logger.error( msg="Не удалось обработать коммиты", - extra=extra_info | {'ex': str(e)}, + extra=extra_info | {"ex": str(e)}, ) return create_contributions( repo, contrib_data, - user_field='author', - id_field='sha', - type_='cit', + user_field="author", + id_field="sha", + type_="cit", ) def _process_comments(self, owner, repo, extra_info): @@ -317,13 +355,13 @@ def _process_comments(self, owner, repo, extra_info): except Exception as e: logger.error( msg="Не удалось обработать комментарии", - extra=extra_info | {'ex': str(e)}, + extra=extra_info | {"ex": str(e)}, ) return create_contributions( repo, contrib_data, - user_field='user', - id_field='id', - type_='cnt', + user_field="user", + id_field="id", + type_="cnt", ) diff --git a/contributors/management/commands/updatesuperuser.py b/contributors/management/commands/updatesuperuser.py index 1981683..636f90d 100644 --- a/contributors/management/commands/updatesuperuser.py +++ b/contributors/management/commands/updatesuperuser.py @@ -1,25 +1,29 @@ -from django.core.management.base import BaseCommand from django.contrib.auth import get_user_model +from django.core.management.base import BaseCommand User = get_user_model() class Command(BaseCommand): - help = 'Updates the superuser' + help = "Updates the superuser" def add_arguments(self, parser): - parser.add_argument('--username', required=True) - parser.add_argument('--email', required=True) + parser.add_argument("--username", required=True) + parser.add_argument("--email", required=True) def handle(self, *args, **options): - username = options['username'] - email = options['email'] + username = options["username"] + email = options["email"] try: user = User.objects.get(username=username) user.email = email user.save() - self.stdout.write(self.style.SUCCESS(f'Superuser {username} updated successfully')) + self.stdout.write( + self.style.SUCCESS(f"Superuser {username} updated successfully") + ) except User.DoesNotExist: User.objects.create_superuser(username=username, email=email) - self.stdout.write(self.style.SUCCESS(f'Superuser {username} created successfully')) + self.stdout.write( + self.style.SUCCESS(f"Superuser {username} created successfully") + ) diff --git a/contributors/views/config.py b/contributors/views/config.py index f4a3ec9..a2e03f6 100644 --- a/contributors/views/config.py +++ b/contributors/views/config.py @@ -15,54 +15,54 @@ def set_up_context(request): """Set up admin site context.""" context = custom.site.each_context(request) - context['title'] = _("Processing configuration") + context["title"] = _("Processing configuration") return context def show_repos(request): """Get repositories for organizations.""" context = set_up_context(request) - if request.method == 'POST': + if request.method == "POST": form_orgs = OrgNamesForm(request.POST) if form_orgs.is_valid(): session = requests.Session() repos_choices = [] - for org_name in form_orgs.cleaned_data['organizations'].split(): + for org_name in form_orgs.cleaned_data["organizations"].split(): org_data = github.get_org_data(org_name, session) - org, _ = misc.update_or_create_record(Organization, org_data) + misc.update_or_create_record(Organization, org_data) repos_data = list(github.get_org_repos(org_name, session)) for repo_data in repos_data: misc.update_or_create_record( Repository, repo_data, - {'is_tracked': False, 'is_visible': False}, + {"is_tracked": False, "is_visible": False}, ) repos_choices.append( - (repo_data['id'], repo_data['name']), + (repo_data["id"], repo_data["name"]), ) session.close() form_repos = RepoNamesForm(choices=repos_choices) - context['form_repos'] = form_repos + context["form_repos"] = form_repos else: form_orgs = OrgNamesForm() - context['form_orgs'] = form_orgs + context["form_orgs"] = form_orgs - return TemplateResponse(request, 'admin/configuration.html', context) + return TemplateResponse(request, "admin/configuration.html", context) def collect_data(request): """Collect data for chosen repositories.""" context = set_up_context(request) - if request.method == 'POST': - repo_ids = request.POST.getlist('repositories') + if request.method == "POST": + repo_ids = request.POST.getlist("repositories") repos = Repository.objects.filter(id__in=repo_ids) for repo in repos: repo.is_tracked = True repo.is_visible = True - Repository.objects.bulk_update(repos, ['is_tracked', 'is_visible']) - fetch_command = ['./manage.py', 'fetchdata', '--repo'] + Repository.objects.bulk_update(repos, ["is_tracked", "is_visible"]) + fetch_command = ["./manage.py", "fetchdata", "--repo"] fetch_command.extend([repo.full_name for repo in repos]) # noqa: E501 subprocess.Popen(fetch_command) # noqa: S603 - return TemplateResponse(request, 'admin/data_collection.html', context) + return TemplateResponse(request, "admin/data_collection.html", context) return HttpResponseForbidden("Forbidden.") diff --git a/pyproject.toml b/pyproject.toml index 8cd1b71..b59e91b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,13 +24,13 @@ dependencies = [ "python-dateutil>=2.9.0.post0", "python-dotenv>=1.0.1", "requests>=2.32.3", - "ruff>=0.7.4", "sentry-sdk>=2.18.0", "whitenoise>=6.8.2", ] [tool.uv] dev-dependencies = [ + "ruff>=0.7.4", "coverage>=7.6.7", "faker>=33.0.0", "pre-commit>=4.0.1", diff --git a/ruff.toml b/ruff.toml index 54dd7b9..c8c80fc 100644 --- a/ruff.toml +++ b/ruff.toml @@ -6,11 +6,12 @@ exclude = [ "__pycache__", "migrations", ".venv", + "settings.py" ] -[lint.mccabe] -# Flag errors (`C901`) whenever the complexity level exceeds 5. -max-complexity = 5 +[lint] +preview = true +select = ["E", "F", "I", "C90"] [lint.extend-per-file-ignores] "__init__.py" = ["F401"] diff --git a/uv.lock b/uv.lock index 871673a..ab7aa39 100644 --- a/uv.lock +++ b/uv.lock @@ -401,7 +401,6 @@ dependencies = [ { name = "python-dateutil" }, { name = "python-dotenv" }, { name = "requests" }, - { name = "ruff" }, { name = "sentry-sdk" }, { name = "whitenoise" }, ] @@ -413,6 +412,7 @@ dev = [ { name = "pre-commit" }, { name = "pydot" }, { name = "pyparsing" }, + { name = "ruff" }, ] [package.metadata] @@ -436,7 +436,6 @@ requires-dist = [ { name = "python-dateutil", specifier = ">=2.9.0.post0" }, { name = "python-dotenv", specifier = ">=1.0.1" }, { name = "requests", specifier = ">=2.32.3" }, - { name = "ruff", specifier = ">=0.7.4" }, { name = "sentry-sdk", specifier = ">=2.18.0" }, { name = "whitenoise", specifier = ">=6.8.2" }, ] @@ -448,6 +447,7 @@ dev = [ { name = "pre-commit", specifier = ">=4.0.1" }, { name = "pydot", specifier = ">=3.0.2" }, { name = "pyparsing", specifier = ">=3.2.0" }, + { name = "ruff", specifier = ">=0.7.4" }, ] [[package]]