Skip to content

Commit

Permalink
Merge pull request #4 from Knuckles-Team/sqlalchemy
Browse files Browse the repository at this point in the history 
All Tested SQLAlchemy Changes
  • Loading branch information
@Knucklessg1
Knucklessg1 authored Aug 6, 2024
2 parents 20d1db6 + 6c7b618 commit 4d7302d
Show file tree
Hide file tree
Showing 7 changed files with 443 additions and 161 deletions.
137 changes: 116 additions & 21 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
![PyPI - Wheel](https://img.shields.io/pypi/wheel/gitlab-api)
![PyPI - Implementation](https://img.shields.io/pypi/implementation/gitlab-api)

*Version: 0.15.60*
*Version: 1.0.0*

Pythonic GitLab API Library

Expand Down Expand Up @@ -50,6 +50,8 @@ This repository is actively maintained - Contributions are welcome!
<details>
<summary><b>Usage:</b></summary>

Using the API directly

```python
#!/usr/bin/python
# coding: utf-8
Expand All @@ -74,30 +76,123 @@ response = client.get_runners(runner_type='instance_type', all_runners=True)
print(f"Runners: {response}")
```

Inserting API responses directly to a Postgres Database

```python
#!/usr/bin/python
# coding: utf-8
import gitlab_api
from gitlab_api import Branch


token = "<GITLAB_TOKEN/PERSONAL_TOKEN>"
gitlab_url = "<GITLAB_URL>"
client = gitlab_api.Api(url=gitlab_url, token=token)

users = client.get_users()
print(users)

created_merge_request = client.create_merge_request(project_id=123, source_branch="development",
target_branch="production",title="Merge Request Title")
print(f"Merge Request Title: {created_merge_request.data.title}\nDescription: {created_merge_request.data.description}")
users = client.get_users()
print(f"First user's email: {users.data[0].email}")

print(f"Projects: {client.get_projects()}")
import gitlab_api
from gitlab_api.utils import upsert
from gitlab_api.gitlab_db_models import BaseDBModel as Base
import urllib3
import os
from urllib.parse import quote_plus
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

gitlab_token = os.environ["GITLAB_TOKEN"]
postgres_username = os.environ["POSTGRES_USERNAME"]
postgres_password = os.environ["POSTGRES_PASSWORD"]
postgres_db_host = os.environ["POSTGRES_DB_HOST"]
postgres_port = os.environ["POSTGRES_PORT"]
postgres_db_name = os.environ["POSTGRES_DB_NAME"]


if __name__ == "__main__":
print("Creating GitLab Client...")
client = gitlab_api.Api(
url="http://gitlab.arpa/api/v4/",
token=gitlab_token,
verify=False,
)
print("GitLab Client Created\n\n")

print("Creating Engine")
engine = create_engine(
f"postgresql://{postgres_username}:{quote_plus(postgres_password)}@"
f"{postgres_db_host}:{postgres_port}/{postgres_db_name}"
)
print("Engine Created\n\n")

print("Creating Tables...")
Base.metadata.create_all(engine)
print("Tables Created\n\n")

print("Creating Session...")
Session = sessionmaker(bind=engine)
session = Session()
print("Session Created\n\n")

print("Fetching GitLab Data...")
# User Data table is a dependency table
user_response = client.get_users()
print(
f"Users ({len(user_response.data.users)}) Fetched - "
f"Status: {user_response.status_code}\n"
)

# Namespaces table is a dependency table
namespace_response = client.get_namespaces()
print(
f"Namespaces ({len(namespace_response.data.namespaces)}) Fetched - "
f"Status: {namespace_response.status_code}\n"
)

# Project table requires Users and Namespaces
project_response = client.get_nested_projects_by_group(group_id=2, per_page=100)
print(
f"Projects ({len(project_response.data.projects)}) Fetched - "
f"Status: {project_response.status_code}\n"
)

# Merge Requests table requires Users, Namespaces, and Projects
merge_request_response = client.get_group_merge_requests(
argument="state=all", group_id=2
)
print(
f"Merge Requests ({len(merge_request_response.data.merge_requests)}) Fetched - "
f"Status: {merge_request_response.status_code}\n\n"
)

pipeline_job_responses = []
for project in project_response.data.projects:
pipeline_job_response = client.get_project_jobs(project_id=49) # project.id)
pipeline_job_responses.append(pipeline_job_response)
print(
f"Pipeline Jobs ({len(pipeline_job_response.data.jobs)}) Fetched for Project ({project.id}) - "
f"Status: {pipeline_job_response.status_code}\n\n"
)
print(
f"Inserting Pipeline Job {pipeline_job_response}\n\n"
f"Data: {pipeline_job_response.data}"
)

print("Inserting Users Into Database...")
upsert(session=session, response=user_response)
print("Users Synchronization Complete!\n")

print("Inserting Namespaces Into Database...")
upsert(session=session, response=namespace_response)
print("Namespaces Synchronization Complete!\n")

print("Inserting Projects Into Database...\n")
upsert(session=session, response=project_response)
print("Projects Synchronization Complete!\n")

print("Inserting Merge Requests Into Database...")
upsert(session=session, response=merge_request_response)
print("Merge Request Synchronization Complete!\n")

print(f"Inserting ({len(pipeline_job_responses)}) Pipeline Jobs Into Database...")
for pipeline_job_response in pipeline_job_responses:
upsert(session=session, response=pipeline_job_response)
print("Pipeline Jobs Synchronization Complete!\n\n\n")

session.close()
print("Session Closed")

response = client.get_runners(runner_type='instance_type', all_runners=True)
print(f"Runners: {response}")
```

</details>
Expand Down
2 changes: 2 additions & 0 deletions gitlab_api/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@
EpicDBModel,
IssueDBModel,
JobDBModel,
ParentIDDBModel,
PipelineDBModel,
PipelineVariableDBModel,
PackageLinkDBModel,
Expand Down Expand Up @@ -320,6 +321,7 @@
"ConfigurationDBModel",
"IterationDBModel",
"IdentityDBModel",
"ParentIDDBModel",
"GroupSamlIdentityDBModel",
"CreatedByDBModel",
"UserDBModel",
Expand Down
94 changes: 75 additions & 19 deletions gitlab_api/gitlab_db_models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -422,7 +422,7 @@ class ReleaseDBModel(BaseDBModel):
)

commit_id = Column(
Integer,
String,
ForeignKey(column="commits.id", name="fk_release_commits"),
nullable=True,
)
Expand Down Expand Up @@ -520,7 +520,7 @@ class BranchDBModel(BaseDBModel):
inherited = Column(Boolean, nullable=True)

commit_id = Column(
Integer,
String,
ForeignKey(column="commits.id", name="fk_branch_commits"),
nullable=True,
)
Expand Down Expand Up @@ -613,17 +613,11 @@ class TopicDBModel(BaseDBModel):
name = Column(String, nullable=False)



tags_association = Table(
"tags_association",
BaseDBModel.metadata,
Column("tag_id", Integer, ForeignKey("tags.id"), primary_key=True),
Column(
"tags_collection",
Integer,
ForeignKey("tags_collection.id"),
primary_key=True,
),
Column("tags_collection_id", Integer, ForeignKey("tags_collection.id")),
Column("tag_id", Integer, ForeignKey("tags.id")),
)


Expand All @@ -632,14 +626,14 @@ class TagDBModel(BaseDBModel):

id = Column(Integer, primary_key=True, autoincrement=True)
base_type = Column(String, default="Tag")
name = Column(String, nullable=False)
tag = Column(String, nullable=False)
name = Column(String, nullable=True)
tag = Column(String, nullable=True)


class TagsDBModel(BaseDBModel):
__tablename__ = "tags_collection"

id = Column(Integer, primary_key=True, autoincrement=True, nullable=True)
id = Column(Integer, primary_key=True, autoincrement=True)
base_type = Column(String, default="Tags")
tags = relationship(
"TagDBModel",
Expand Down Expand Up @@ -1492,7 +1486,7 @@ class JobDBModel(BaseDBModel):
)

commit_id = Column(
Integer, ForeignKey(column="commits.id", name="fk_job_commit"), nullable=True
String, ForeignKey(column="commits.id", name="fk_job_commit"), nullable=True
)
commit = relationship(
argument="CommitDBModel",
Expand Down Expand Up @@ -1561,11 +1555,11 @@ class JobDBModel(BaseDBModel):

artifacts_id = Column(
Integer,
ForeignKey(column="artifacts.id", name="fk_job_artifacts"),
ForeignKey(column="artifacts_collection.id", name="fk_job_artifacts"),
nullable=True,
)
artifacts = relationship(
argument="ArtifactDBModel", backref=backref("jobs_artifacts")
argument="ArtifactsDBModel", backref=backref("jobs_artifacts")
)


Expand Down Expand Up @@ -1793,7 +1787,7 @@ class CommentDBModel(BaseDBModel):
)

commits_id = Column(
Integer,
String,
ForeignKey(column="commits.id", name="fk_comment_commit"),
nullable=True,
)
Expand All @@ -1804,17 +1798,49 @@ class CommentDBModel(BaseDBModel):
)


parent_ids_association = Table(
"parent_ids_association",
BaseDBModel.metadata,
Column("parent_ids_id", Integer, ForeignKey("parent_ids.id"), primary_key=True),
Column(
"parent_ids_collection",
Integer,
ForeignKey("parent_ids_collection.id"),
primary_key=True,
),
)


class ParentIDDBModel(BaseDBModel):
__tablename__ = "parent_ids"

id = Column(Integer, primary_key=True, autoincrement=True)
base_type = Column(String, default="ParentID")
parent_id = Column(String, nullable=False)


class ParentIDsDBModel(BaseDBModel):
__tablename__ = "parent_ids_collection"

id = Column(Integer, primary_key=True, autoincrement=True, nullable=True)
base_type = Column(String, default="ParentIDs")
parent_ids = relationship(
"ParentIDDBModel",
secondary=parent_ids_association,
backref=backref("parent_ids_collection", lazy="dynamic"),
)


# Commit Model
class CommitDBModel(BaseDBModel):
__tablename__ = "commits"

id = Column(Integer, primary_key=True)
id = Column(String, primary_key=True)
base_type = Column(String, default="Commit")
short_id = Column(String, nullable=True)
started_at = Column(DateTime, nullable=True)
finished_at = Column(DateTime, nullable=True)
created_at = Column(DateTime, nullable=True)
parent_ids = Column(JSON, nullable=True)
title = Column(String, nullable=True)
description = Column(String, nullable=True)
message = Column(String, nullable=True)
Expand Down Expand Up @@ -1889,6 +1915,16 @@ class CommitDBModel(BaseDBModel):
foreign_keys=[notes_id],
backref=backref("commit_notes"),
)
parent_ids_id = Column(
Integer,
ForeignKey(column="parent_ids_collection.id", name="fk_commit_parent_ids"),
nullable=True,
)
parent_ids = relationship(
argument="ParentIDsDBModel",
foreign_keys=[parent_ids_id],
backref=backref("commit_parent_ids"),
)


# Membership Model
Expand Down Expand Up @@ -2028,6 +2064,14 @@ class ReferencesDBModel(BaseDBModel):
full = Column(String, nullable=True)


artifacts_association = Table(
"artifacts_association",
BaseDBModel.metadata,
Column("artifacts_collection_id", Integer, ForeignKey("artifacts_collection.id")),
Column("artifact_id", Integer, ForeignKey("artifacts.id")),
)


# Artifact Model
class ArtifactDBModel(BaseDBModel):
__tablename__ = "artifacts"
Expand All @@ -2040,6 +2084,18 @@ class ArtifactDBModel(BaseDBModel):
file_format = Column(String, nullable=True)


class ArtifactsDBModel(BaseDBModel):
__tablename__ = "artifacts_collection"

id = Column(Integer, primary_key=True, autoincrement=True, nullable=True)
base_type = Column(String, default="Artifacts")
artifacts = relationship(
"ArtifactDBModel",
secondary=artifacts_association,
backref=backref("artifacts_collection", lazy="dynamic"),
)


# ArtifactsFile Model
class ArtifactsFileDBModel(BaseDBModel):
__tablename__ = "artifacts_files"
Expand Down
Loading

0 comments on commit 4d7302d

Please sign in to comment.