Skip to content

Commit

Permalink
fix build system
Browse files Browse the repository at this point in the history
  • Loading branch information
EthanLeitch committed Nov 7, 2024
1 parent 08b19b2 commit d72c14f
Show file tree
Hide file tree
Showing 6 changed files with 34 additions and 38 deletions.
2 changes: 1 addition & 1 deletion ao3scraper/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = '1.0.2'
__version__ = '1.0.3'
28 changes: 14 additions & 14 deletions ao3scraper/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@
import warnings

# Custom modules
import ao3scraper.constants as constants
import ao3scraper.database as database
import constants
import database

# Create columns of rich table
table = Table(title="Fanfics", show_lines=True)
Expand Down Expand Up @@ -88,7 +88,7 @@ def scrape_urls():
exit()
else:
print("Contacted servers successfully.")

# Create array of None that will be replaced later depending on the thread's index.
external_fics = [None for fic in fic_ids]

Expand Down Expand Up @@ -119,11 +119,11 @@ def custom_formatwarning(msg, *args, **kwargs):
e = "The work might be restricted (AttributeError)"

external_fics[count] = ({'Exception': str(e), 'id': id})

progress.update(progress_bar, advance=1)

progress.update(progress_bar, advance=1)

# Track and create thread pool
with Progress() as progress:
progress_bar = progress.add_task("Fetching data from AO3...", total=len(fic_ids))
Expand All @@ -133,7 +133,7 @@ def custom_formatwarning(msg, *args, **kwargs):

# Handle adding of each fic
for count, fic in enumerate(external_fics):

if 'Exception' in fic:
add_row(fic, count)
continue
Expand All @@ -142,10 +142,10 @@ def custom_formatwarning(msg, *args, **kwargs):
# If type of entry is list, store it as comma-seperated string (e.g. "foo, bar").
if type(fic[value]) is list:
fic[value] = ", ".join(fic[value])

# Strip leading and trailing whitespace from fic summaries.
fic['summary'] = fic['summary'].strip()

if type(local_fics[count]['nchapters']) is type(None):
add_row(fic, count)
elif int(fic['nchapters']) > int(local_fics[count]['nchapters']):
Expand Down Expand Up @@ -222,7 +222,7 @@ def construct_rich_table(read_again=True):
if read_again is True:
# Read database again
local_fics = database.get_all_fics()

for count, fic in enumerate(local_fics):
add_row(fic, count)

Expand All @@ -246,7 +246,7 @@ def add_row(fic, count, styling=""):
index = str(count + 1)

# Create AO3 link for fic
fic_link = f"https://archiveofourown.org/works/{fic['id']}"
fic_link = f"https://archiveofourown.org/works/{fic['id']}"

# If key 'Exception' in fic, display error information.
if 'Exception' in fic:
Expand All @@ -261,8 +261,8 @@ def add_row(fic, count, styling=""):
# Converting expected_chapters from None to '?' makes the "Chapters" column look nicer.
if fic['expected_chapters'] is None or fic['expected_chapters'] == "None":
fic['expected_chapters'] = '?'
# Turn upload date of fic into correct format

# Turn upload date of fic into correct format
then = datetime.strptime(fic['date_updated'], constants.DATE_FORMAT)

# Shorten date strings from YYYY-MM-DD XX:XX:XX to YYYY-MM-DD.
Expand Down Expand Up @@ -292,7 +292,7 @@ def add_row(fic, count, styling=""):
else:
new_args.append(fic[constants.TABLE_TEMPLATE[count]['column']])
new_args.insert(0, f"{index}.")

if (constants.NOW - then).days > constants.STALE_THRESHOLD:
table.add_row(*new_args, style=constants.STALE_STYLES)
else:
Expand Down
8 changes: 4 additions & 4 deletions ao3scraper/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,12 @@
from yaml import load, Loader

# Custom modules
import ao3scraper.file_validator as file_validator
import file_validator

# Set important constants!
APP_NAME = "ao3scraper"
APP_AUTHOR = "EthanLeitch"
APP_VERSION = metadata.version(APP_NAME)
APP_VERSION = '1.0.3' #metadata.version(APP_NAME)
ALEMBIC_VERSION = 'ca2dfefaf0b6'

DATA_PATH = path.join(user_data_dir(APP_NAME, APP_AUTHOR)) + "/"
Expand Down Expand Up @@ -52,7 +52,7 @@
"""
Currently this list is only avaliable by fetching an instance of a Work's metadata (AO3.Work.metadata).
This is done in construct_list.py
'id' has been excluded from this list to prevent conflicts with the SQLAlchemy primary_key.
'id' has been excluded from this list to prevent conflicts with the SQLAlchemy primary_key.
"""
TABLE_COLUMNS = ['date_edited', 'date_published', 'date_updated', 'bookmarks', 'categories', 'nchapters', 'characters', 'complete', 'comments', 'expected_chapters', 'fandoms', 'hits', 'kudos', 'language', 'rating', 'relationships', 'restricted', 'status', 'summary', 'tags', 'title', 'warnings', 'words', 'collections', 'authors', 'series', 'chapter_titles']
CUSTOM_COLUMNS = ['$chapters', '$latest_chapter']
Expand Down Expand Up @@ -81,4 +81,4 @@
MARKER = "# Enter one url on each line to add it to the database. This line will not be recorded."

DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
NOW = datetime.now()
NOW = datetime.now()
9 changes: 4 additions & 5 deletions ao3scraper/database.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@
from sqlalchemy.orm import Session, sessionmaker
from sqlalchemy import inspect, select, update, delete, values

# marshmallow
# marshmallow
from marshmallow_sqlalchemy import SQLAlchemyAutoSchema

# Custom modules
import ao3scraper.constants as constants
import constants

engine = db.create_engine(f'sqlite:///{constants.DATABASE_FILE_PATH}')
connection = engine.connect()
Expand All @@ -28,7 +28,7 @@
# Create fanfic class
class Fanfic(Base):
__tablename__ = "fics"

id = db.Column(Integer, primary_key=True, unique=True)

# Add each item in TABLE_COLUMNS to the database as a db.Column(String) object.
Expand Down Expand Up @@ -91,7 +91,7 @@ def get_all_fics():
query = session.query(Fanfic)
result_dict = [FanficSchema().dump(u) for u in query.all()]
return result_dict


def get_fic_ids():
"""Returns all fic IDs from the database in a list."""
Expand All @@ -100,4 +100,3 @@ def get_fic_ids():
query = session.query(Fanfic.id).all()
result = [r for r, in query]
return result

16 changes: 8 additions & 8 deletions ao3scraper/file_validator.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@
from shutil import copy

# Import custom modules
import ao3scraper.constants as constants
import constants

def main():
# Create config file if config file does not exist
if not path.exists(constants.CONFIG_FILE_PATH):
print("No config file found. Creating new config file...")

pathlib.Path(constants.CONFIG_PATH).mkdir(parents=True, exist_ok=True)

with open(constants.CONFIG_FILE_PATH, 'w') as file:
# Convert CONFIG_TEMPLATE to yaml, and disable the alphabetical key sorting done by yaml.dump
#config_file_dump = dump(constants.CONFIG_TEMPLATE, Dumper=Dumper, sort_keys=False)
Expand All @@ -40,7 +40,7 @@ def main():
print("No database found. Creating new database...")

pathlib.Path(constants.DATA_PATH).mkdir(parents=True, exist_ok=True)

# Connect to database
connection = sqlite3.connect(constants.DATABASE_FILE_PATH)
cursor = connection.cursor()
Expand All @@ -49,7 +49,7 @@ def main():
cursor.execute("CREATE TABLE fics (id INTEGER)")
for column in constants.TABLE_COLUMNS:
cursor.execute(f"ALTER TABLE fics ADD {column} TEXT")

# Create metadata table
cursor.execute("CREATE TABLE alembic_version (version_num VARCHAR(32) NOT NULL, CONSTRAINT alembic_version_pkc PRIMARY KEY (version_num));")
query = f"INSERT INTO alembic_version VALUES ('{constants.ALEMBIC_VERSION}');"
Expand All @@ -59,7 +59,7 @@ def main():
connection.close()

print("Database created.\n")

validate_database()

def validate_database():
Expand All @@ -83,10 +83,10 @@ def validate_database():
if version_num != constants.ALEMBIC_VERSION:
print(f"ao3scraper is on database revision {constants.ALEMBIC_VERSION}, but the local database is on revision {version_num}.")
upgrade_database()

def upgrade_database():
print("Would you like to migrate to the version supported by ao3scraper? (y/n)")

choice = input(" > ").lower()
if choice == 'y':
backup_db_name = "fics_backup_" + str(uuid.uuid4().hex) + ".db"
Expand All @@ -101,4 +101,4 @@ def upgrade_database():
quit()
else:
# Quitting is the safest option, as ao3scraper doesn't do error handling for the database very well.
quit()
quit()
9 changes: 3 additions & 6 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,15 +1,12 @@
[tool.poetry]
name = "ao3scraper"
version = "1.0.2"
version = "1.0.3"
readme = "README.md"
description = "ao3scraper is a python webscraper that scrapes AO3 for fanfiction data, stores it in a database, and highlights entries when they are updated."
license = "GPL-3.0-or-later"
authors = ["Ethan <[email protected]>"]
repository = "https://github.com/EthanLeitch/ao3scraper"
packages = [
{ include = "ao3scraper" },
{ include = "ao3scraper/**/*.py" },
]
packages = [{ include = "ao3scraper" }, { include = "ao3scraper/**/*.py" }]

[tool.poetry.scripts]
ao3scraper = "ao3scraper.cli:main"
Expand All @@ -30,7 +27,7 @@ urllib3 = "1.26.19"
configparser = "^5.3.0"
SQLAlchemy = "^1.4.41"
marshmallow-sqlalchemy = "^0.28.1"
deepdiff = {extras = ["murmur"], version = "^5.8.1"}
deepdiff = { extras = ["murmur"], version = "^5.8.1" }
dictdiffer = "^0.9.0"
pathlib = "^1.0.1"
platformdirs = "^2.5.4"
Expand Down

0 comments on commit d72c14f

Please sign in to comment.