Skip to content

Commit

Permalink
fix ruff config
Browse files Browse the repository at this point in the history
  • Loading branch information
sgmdlt committed Dec 5, 2024
1 parent 402d327 commit 90b80f0
Show file tree
Hide file tree
Showing 6 changed files with 138 additions and 95 deletions.
172 changes: 105 additions & 67 deletions contributors/management/commands/fetchdata.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
import logging
import sys

from django.db import transaction

import requests
from django.core import management
from django.db import transaction
from django.utils import dateparse

from contributors.models import (
Expand All @@ -21,7 +20,7 @@
from contributors.utils import misc

# Simultaneous logging to file and stdout
logger = logging.getLogger('GitHub')
logger = logging.getLogger("GitHub")
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler(sys.stdout))

Expand All @@ -37,7 +36,9 @@


@transaction.atomic
def create_contributions(repo, contrib_data, user_field='user', id_field='id', type_='iss'): # noqa: C901
def create_contributions(
repo, contrib_data, user_field="user", id_field="id", type_="iss"
): # noqa: C901
"""
Создает записи о контрибуциях в базе данных.
"""
Expand All @@ -51,21 +52,27 @@ def create_contributions(repo, contrib_data, user_field='user', id_field='id', t
logger.info(contrib)
contributor = None
if _is_valid_contributor(contrib, user_field):
login = contrib[user_field]['login']
login = contrib[user_field]["login"]
if login not in contributors:
contributor, created = Contributor.objects.get_or_create(login=login)
contributor, _ = Contributor.objects.get_or_create(login=login)
contributors[login] = contributor
else:
contributor = contributors[login]

if contributor:
contribution = _create_contribution(repo, contrib, contributor, id_field, type_)
contribution = _create_contribution(
repo, contrib, contributor, id_field, type_
)
contributions.append(contribution)

if type_ == 'cit':
commit_stats.append(_create_commit_stats(repo, contribution, contrib, id_field))
elif type_ in {'pr', 'iss'}:
issue_infos.append(_create_issue_info(repo, contribution, contrib, type_))
if type_ == "cit":
commit_stats.append(
_create_commit_stats(repo, contribution, contrib, id_field)
)
elif type_ in {"pr", "iss"}:
issue_infos.append(
_create_issue_info(repo, contribution, contrib, type_)
)
contribution_labels.extend(_create_contribution_labels(contrib))

_bulk_create_records(contributions, commit_stats, issue_infos, contribution_labels)
Expand All @@ -75,52 +82,65 @@ def create_contributions(repo, contrib_data, user_field='user', id_field='id', t
def _is_valid_contributor(contrib, user_field):
"""Проверяет, является ли контрибуция допустимой для обработки."""
contrib_author = contrib[user_field]
return (contrib_author is not None
and contrib_author['type'] != 'Bot'
and contrib_author['login'] not in IGNORED_CONTRIBUTORS)
return (
contrib_author is not None
and contrib_author["type"] != "Bot"
and contrib_author["login"] not in IGNORED_CONTRIBUTORS
)


def _create_contribution(repo, contrib, contributor, id_field, type_):
"""Создает объект Contribution."""
true_type = 'pr' if (type_ == 'iss' and 'pull_request' in contrib) else type_
datetime = contrib['commit']['author']['date'] if true_type == 'cit' else contrib['created_at']
true_type = "pr" if (type_ == "iss" and "pull_request" in contrib) else type_
datetime = (
contrib["commit"]["author"]["date"]
if true_type == "cit"
else contrib["created_at"]
)

return Contribution(
id=contrib[id_field],
repository=repo,
contributor=contributor,
type=true_type,
html_url=contrib['html_url'],
html_url=contrib["html_url"],
created_at=dateparse.parse_datetime(datetime),
)


def _create_commit_stats(repo, contribution, contrib, id_field):
"""Создает объект CommitStats для коммита."""
commit_data = github.get_commit_data(repo.organization or repo.owner, repo, contrib[id_field], session)
commit_data = github.get_commit_data(
repo.organization or repo.owner, repo, contrib[id_field], session
)
return CommitStats(
commit=contribution,
additions=commit_data['stats']['additions'],
deletions=commit_data['stats']['deletions'],
additions=commit_data["stats"]["additions"],
deletions=commit_data["stats"]["deletions"],
)


def _create_issue_info(repo, contribution, contrib, type_):
"""Создает объект IssueInfo для issue или pull request."""
state = 'merged' if type_ == 'pr' and github.is_pr_merged(
repo.organization or repo.owner, repo, contrib['number'], session
) else contrib['state']
state = (
"merged"
if type_ == "pr"
and github.is_pr_merged(
repo.organization or repo.owner, repo, contrib["number"], session
)
else contrib["state"]
)

return IssueInfo(
issue=contribution,
title=contrib['title'],
title=contrib["title"],
state=state,
)


def _create_contribution_labels(contrib):
"""Создает объекты ContributionLabel для вклада."""
return [ContributionLabel(name=label["name"]) for label in contrib['labels']]
return [ContributionLabel(name=label["name"]) for label in contrib["labels"]]


def _bulk_create_records(contributions, commit_stats, issue_infos, contribution_labels):
Expand All @@ -134,9 +154,13 @@ def _bulk_create_records(contributions, commit_stats, issue_infos, contribution_
@transaction.atomic
def _link_labels_to_contributions(contributions, contrib_data):
"""Связывает метки с контрибуциями."""
all_label_names = set(label['name'] for c in contrib_data for label in c['labels'])
existing_labels = {label.name: label for label in Label.objects.filter(name__in=all_label_names)}
new_labels = [Label(name=name) for name in all_label_names if name not in existing_labels]
all_label_names = set(label["name"] for c in contrib_data for label in c["labels"])
existing_labels = {
label.name: label for label in Label.objects.filter(name__in=all_label_names)
}
new_labels = [
Label(name=name) for name in all_label_names if name not in existing_labels
]
Label.objects.bulk_create(new_labels, ignore_conflicts=True)

all_labels = existing_labels.copy()
Expand All @@ -145,11 +169,16 @@ def _link_labels_to_contributions(contributions, contrib_data):
ContributionLabel = Contribution.labels.through
contribution_labels = []

for contribution, labels in zip(contributions, [c['labels'] for c in contrib_data]):
contribution_labels.extend([
ContributionLabel(contribution_id=contribution.id, label_id=all_labels[label['name']].id)
for label in labels
])
for contribution, labels in zip(contributions, [c["labels"] for c in contrib_data]):
contribution_labels.extend(
[
ContributionLabel(
contribution_id=contribution.id,
label_id=all_labels[label["name"]].id,
)
for label in labels
]
)

# Удаляем существующие связи
ContributionLabel.objects.filter(contribution__in=contributions).delete()
Expand All @@ -166,13 +195,15 @@ class Command(management.base.BaseCommand):
def add_arguments(self, parser):
"""Добавляет аргументы для команды."""
parser.add_argument(
'owner',
nargs='*',
"owner",
nargs="*",
default=ORGANIZATIONS,
help='список имен владельцев',
help="список имен владельцев",
)
parser.add_argument(
'--repo', nargs='*', help='список полных имен репозиториев',
"--repo",
nargs="*",
help="список полных имен репозиториев",
)

def handle(self, *args, **options):
Expand All @@ -187,19 +218,21 @@ def handle(self, *args, **options):
finally:
session.close()

logger.info(self.style.SUCCESS(
"Данные получены из GitHub и сохранены в базу данных",
))
logger.info(
self.style.SUCCESS(
"Данные получены из GitHub и сохранены в базу данных",
)
)

def _get_data_of_owners_and_repos(self, options):
"""Получает данные о владельцах и репозиториях."""
if options['repo']:
if options["repo"]:
return github.get_data_of_owners_and_repos(
repo_full_names=options['repo'],
repo_full_names=options["repo"],
)
elif options['owner']:
elif options["owner"]:
return github.get_data_of_owners_and_repos(
owner_names=options['owner'],
owner_names=options["owner"],
)
else:
raise management.base.CommandError(
Expand All @@ -217,15 +250,15 @@ def _process_owners_and_repos(self, data_of_owners_and_repos):

def _process_owner(self, owner_data):
"""Обрабатывает данные о владельце."""
owner_type = owner_data['details']['type']
table = Contributor if owner_type == 'User' else Organization
owner, _ = misc.update_or_create_record(table, owner_data['details'])
owner_type = owner_data["details"]["type"]
table = Contributor if owner_type == "User" else Organization
owner, _ = misc.update_or_create_record(table, owner_data["details"])
logger.info(f"Обработка {owner_type}: {owner}")
return owner

def _process_organization_repos(self, organization, org_data):
"""Обрабатывает репозитории организации."""
repos_data = org_data.get('repos', [])
repos_data = org_data.get("repos", [])
number_of_repos = len(repos_data)
for i, repo_data in enumerate(repos_data, start=1):
self._process_repo(organization, repo_data, i, number_of_repos)
Expand All @@ -239,15 +272,18 @@ def _process_user_repos(self, user):

def _get_repos_to_process(self, owner):
"""Получает список репозиториев для обработки."""
return Repository.objects.filter(owner=owner).exclude(
name__in=IGNORED_REPOSITORIES
).select_related('owner').prefetch_related('labels')
return (
Repository.objects.filter(owner=owner)
.exclude(name__in=IGNORED_REPOSITORIES)
.select_related("owner")
.prefetch_related("labels")
)

def _process_repo(self, owner, repo_data, i, number_of_repos):
"""Обрабатывает отдельный репозиторий."""
repo, _ = misc.update_or_create_record(Repository, repo_data)
logger.info(f"Обработка репозитория: {repo} ({i}/{number_of_repos})")
if repo_data['size'] == 0:
if repo_data["size"] == 0:
logger.info("Пустой репозиторий, пропускаем")
return

Expand All @@ -256,14 +292,14 @@ def _process_repo(self, owner, repo_data, i, number_of_repos):

def _process_repo_language(self, repo, repo_data):
"""Обрабатывает язык репозитория."""
language = repo_data['language']
language = repo_data["language"]
if language:
label, _ = Label.objects.get_or_create(name=language)
repo.labels.add(label)

def _process_repo_contributions(self, owner, repo):
"""Обрабатывает вклады в репозиторий."""
extra_info = {'owner': owner, 'repo': repo}
extra_info = {"owner": owner, "repo": repo}
self._process_issues_and_prs(owner, repo, extra_info)
self._process_commits(owner, repo, extra_info)
self._process_comments(owner, repo, extra_info)
Expand All @@ -276,37 +312,39 @@ def _process_issues_and_prs(self, owner, repo, extra_info):
logger.info(contrib_data)
except Exception as e:
logger.error(
extra=extra_info | {'ex': str(e)},
extra=extra_info | {"ex": str(e)},
msg="Не удалось обработать issues и pull requests",
)
return
create_contributions(
repo,
contrib_data,
user_field='user',
id_field='id',
type_='iss',
user_field="user",
id_field="id",
type_="iss",
)

def _process_commits(self, owner, repo, extra_info):
"""Обрабатывает коммиты."""
logger.info("Обработка коммитов")
try:
contrib_data = github.get_repo_commits_except_merges(
owner, repo, session=session,
owner,
repo,
session=session,
)
except Exception as e:
logger.error(
msg="Не удалось обработать коммиты",
extra=extra_info | {'ex': str(e)},
extra=extra_info | {"ex": str(e)},
)
return
create_contributions(
repo,
contrib_data,
user_field='author',
id_field='sha',
type_='cit',
user_field="author",
id_field="sha",
type_="cit",
)

def _process_comments(self, owner, repo, extra_info):
Expand All @@ -317,13 +355,13 @@ def _process_comments(self, owner, repo, extra_info):
except Exception as e:
logger.error(
msg="Не удалось обработать комментарии",
extra=extra_info | {'ex': str(e)},
extra=extra_info | {"ex": str(e)},
)
return
create_contributions(
repo,
contrib_data,
user_field='user',
id_field='id',
type_='cnt',
user_field="user",
id_field="id",
type_="cnt",
)
20 changes: 12 additions & 8 deletions contributors/management/commands/updatesuperuser.py
Original file line number Diff line number Diff line change
@@ -1,25 +1,29 @@
from django.core.management.base import BaseCommand
from django.contrib.auth import get_user_model
from django.core.management.base import BaseCommand

User = get_user_model()


class Command(BaseCommand):
help = 'Updates the superuser'
help = "Updates the superuser"

def add_arguments(self, parser):
parser.add_argument('--username', required=True)
parser.add_argument('--email', required=True)
parser.add_argument("--username", required=True)
parser.add_argument("--email", required=True)

def handle(self, *args, **options):
username = options['username']
email = options['email']
username = options["username"]
email = options["email"]

try:
user = User.objects.get(username=username)
user.email = email
user.save()
self.stdout.write(self.style.SUCCESS(f'Superuser {username} updated successfully'))
self.stdout.write(
self.style.SUCCESS(f"Superuser {username} updated successfully")
)
except User.DoesNotExist:
User.objects.create_superuser(username=username, email=email)
self.stdout.write(self.style.SUCCESS(f'Superuser {username} created successfully'))
self.stdout.write(
self.style.SUCCESS(f"Superuser {username} created successfully")
)
Loading

0 comments on commit 90b80f0

Please sign in to comment.