diff --git a/ao3scraper/__init__.py b/ao3scraper/__init__.py index a6221b3..3f6fab6 100644 --- a/ao3scraper/__init__.py +++ b/ao3scraper/__init__.py @@ -1 +1 @@ -__version__ = '1.0.2' +__version__ = '1.0.3' diff --git a/ao3scraper/__main__.py b/ao3scraper/__main__.py index 8a53c90..38d5687 100644 --- a/ao3scraper/__main__.py +++ b/ao3scraper/__main__.py @@ -23,8 +23,8 @@ import warnings # Custom modules -import ao3scraper.constants as constants -import ao3scraper.database as database +import constants +import database # Create columns of rich table table = Table(title="Fanfics", show_lines=True) @@ -88,7 +88,7 @@ def scrape_urls(): exit() else: print("Contacted servers successfully.") - + # Create array of None that will be replaced later depending on the thread's index. external_fics = [None for fic in fic_ids] @@ -119,11 +119,11 @@ def custom_formatwarning(msg, *args, **kwargs): e = "The work might be restricted (AttributeError)" external_fics[count] = ({'Exception': str(e), 'id': id}) - + progress.update(progress_bar, advance=1) - + progress.update(progress_bar, advance=1) - + # Track and create thread pool with Progress() as progress: progress_bar = progress.add_task("Fetching data from AO3...", total=len(fic_ids)) @@ -133,7 +133,7 @@ def custom_formatwarning(msg, *args, **kwargs): # Handle adding of each fic for count, fic in enumerate(external_fics): - + if 'Exception' in fic: add_row(fic, count) continue @@ -142,10 +142,10 @@ def custom_formatwarning(msg, *args, **kwargs): # If type of entry is list, store it as comma-seperated string (e.g. "foo, bar"). if type(fic[value]) is list: fic[value] = ", ".join(fic[value]) - + # Strip leading and trailing whitespace from fic summaries. fic['summary'] = fic['summary'].strip() - + if type(local_fics[count]['nchapters']) is type(None): add_row(fic, count) elif int(fic['nchapters']) > int(local_fics[count]['nchapters']): @@ -222,7 +222,7 @@ def construct_rich_table(read_again=True): if read_again is True: # Read database again local_fics = database.get_all_fics() - + for count, fic in enumerate(local_fics): add_row(fic, count) @@ -246,7 +246,7 @@ def add_row(fic, count, styling=""): index = str(count + 1) # Create AO3 link for fic - fic_link = f"https://archiveofourown.org/works/{fic['id']}" + fic_link = f"https://archiveofourown.org/works/{fic['id']}" # If key 'Exception' in fic, display error information. if 'Exception' in fic: @@ -261,8 +261,8 @@ def add_row(fic, count, styling=""): # Converting expected_chapters from None to '?' makes the "Chapters" column look nicer. if fic['expected_chapters'] is None or fic['expected_chapters'] == "None": fic['expected_chapters'] = '?' - - # Turn upload date of fic into correct format + + # Turn upload date of fic into correct format then = datetime.strptime(fic['date_updated'], constants.DATE_FORMAT) # Shorten date strings from YYYY-MM-DD XX:XX:XX to YYYY-MM-DD. @@ -292,7 +292,7 @@ def add_row(fic, count, styling=""): else: new_args.append(fic[constants.TABLE_TEMPLATE[count]['column']]) new_args.insert(0, f"{index}.") - + if (constants.NOW - then).days > constants.STALE_THRESHOLD: table.add_row(*new_args, style=constants.STALE_STYLES) else: diff --git a/ao3scraper/constants.py b/ao3scraper/constants.py index 9e20c03..2ab2fe5 100644 --- a/ao3scraper/constants.py +++ b/ao3scraper/constants.py @@ -8,12 +8,12 @@ from yaml import load, Loader # Custom modules -import ao3scraper.file_validator as file_validator +import file_validator # Set important constants! APP_NAME = "ao3scraper" APP_AUTHOR = "EthanLeitch" -APP_VERSION = metadata.version(APP_NAME) +APP_VERSION = '1.0.3' #metadata.version(APP_NAME) ALEMBIC_VERSION = 'ca2dfefaf0b6' DATA_PATH = path.join(user_data_dir(APP_NAME, APP_AUTHOR)) + "/" @@ -52,7 +52,7 @@ """ Currently this list is only avaliable by fetching an instance of a Work's metadata (AO3.Work.metadata). This is done in construct_list.py -'id' has been excluded from this list to prevent conflicts with the SQLAlchemy primary_key. +'id' has been excluded from this list to prevent conflicts with the SQLAlchemy primary_key. """ TABLE_COLUMNS = ['date_edited', 'date_published', 'date_updated', 'bookmarks', 'categories', 'nchapters', 'characters', 'complete', 'comments', 'expected_chapters', 'fandoms', 'hits', 'kudos', 'language', 'rating', 'relationships', 'restricted', 'status', 'summary', 'tags', 'title', 'warnings', 'words', 'collections', 'authors', 'series', 'chapter_titles'] CUSTOM_COLUMNS = ['$chapters', '$latest_chapter'] @@ -81,4 +81,4 @@ MARKER = "# Enter one url on each line to add it to the database. This line will not be recorded." DATE_FORMAT = "%Y-%m-%d %H:%M:%S" -NOW = datetime.now() \ No newline at end of file +NOW = datetime.now() diff --git a/ao3scraper/database.py b/ao3scraper/database.py index 915d9ee..5c93c95 100644 --- a/ao3scraper/database.py +++ b/ao3scraper/database.py @@ -10,11 +10,11 @@ from sqlalchemy.orm import Session, sessionmaker from sqlalchemy import inspect, select, update, delete, values -# marshmallow +# marshmallow from marshmallow_sqlalchemy import SQLAlchemyAutoSchema # Custom modules -import ao3scraper.constants as constants +import constants engine = db.create_engine(f'sqlite:///{constants.DATABASE_FILE_PATH}') connection = engine.connect() @@ -28,7 +28,7 @@ # Create fanfic class class Fanfic(Base): __tablename__ = "fics" - + id = db.Column(Integer, primary_key=True, unique=True) # Add each item in TABLE_COLUMNS to the database as a db.Column(String) object. @@ -91,7 +91,7 @@ def get_all_fics(): query = session.query(Fanfic) result_dict = [FanficSchema().dump(u) for u in query.all()] return result_dict - + def get_fic_ids(): """Returns all fic IDs from the database in a list.""" @@ -100,4 +100,3 @@ def get_fic_ids(): query = session.query(Fanfic.id).all() result = [r for r, in query] return result - diff --git a/ao3scraper/file_validator.py b/ao3scraper/file_validator.py index 5337560..0ad98a3 100644 --- a/ao3scraper/file_validator.py +++ b/ao3scraper/file_validator.py @@ -13,7 +13,7 @@ from shutil import copy # Import custom modules -import ao3scraper.constants as constants +import constants def main(): # Create config file if config file does not exist @@ -21,7 +21,7 @@ def main(): print("No config file found. Creating new config file...") pathlib.Path(constants.CONFIG_PATH).mkdir(parents=True, exist_ok=True) - + with open(constants.CONFIG_FILE_PATH, 'w') as file: # Convert CONFIG_TEMPLATE to yaml, and disable the alphabetical key sorting done by yaml.dump #config_file_dump = dump(constants.CONFIG_TEMPLATE, Dumper=Dumper, sort_keys=False) @@ -40,7 +40,7 @@ def main(): print("No database found. Creating new database...") pathlib.Path(constants.DATA_PATH).mkdir(parents=True, exist_ok=True) - + # Connect to database connection = sqlite3.connect(constants.DATABASE_FILE_PATH) cursor = connection.cursor() @@ -49,7 +49,7 @@ def main(): cursor.execute("CREATE TABLE fics (id INTEGER)") for column in constants.TABLE_COLUMNS: cursor.execute(f"ALTER TABLE fics ADD {column} TEXT") - + # Create metadata table cursor.execute("CREATE TABLE alembic_version (version_num VARCHAR(32) NOT NULL, CONSTRAINT alembic_version_pkc PRIMARY KEY (version_num));") query = f"INSERT INTO alembic_version VALUES ('{constants.ALEMBIC_VERSION}');" @@ -59,7 +59,7 @@ def main(): connection.close() print("Database created.\n") - + validate_database() def validate_database(): @@ -83,10 +83,10 @@ def validate_database(): if version_num != constants.ALEMBIC_VERSION: print(f"ao3scraper is on database revision {constants.ALEMBIC_VERSION}, but the local database is on revision {version_num}.") upgrade_database() - + def upgrade_database(): print("Would you like to migrate to the version supported by ao3scraper? (y/n)") - + choice = input(" > ").lower() if choice == 'y': backup_db_name = "fics_backup_" + str(uuid.uuid4().hex) + ".db" @@ -101,4 +101,4 @@ def upgrade_database(): quit() else: # Quitting is the safest option, as ao3scraper doesn't do error handling for the database very well. - quit() \ No newline at end of file + quit() diff --git a/pyproject.toml b/pyproject.toml index 42fb4e7..813600b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,15 +1,12 @@ [tool.poetry] name = "ao3scraper" -version = "1.0.2" +version = "1.0.3" readme = "README.md" description = "ao3scraper is a python webscraper that scrapes AO3 for fanfiction data, stores it in a database, and highlights entries when they are updated." license = "GPL-3.0-or-later" authors = ["Ethan "] repository = "https://github.com/EthanLeitch/ao3scraper" -packages = [ - { include = "ao3scraper" }, - { include = "ao3scraper/**/*.py" }, -] +packages = [{ include = "ao3scraper" }, { include = "ao3scraper/**/*.py" }] [tool.poetry.scripts] ao3scraper = "ao3scraper.cli:main" @@ -30,7 +27,7 @@ urllib3 = "1.26.19" configparser = "^5.3.0" SQLAlchemy = "^1.4.41" marshmallow-sqlalchemy = "^0.28.1" -deepdiff = {extras = ["murmur"], version = "^5.8.1"} +deepdiff = { extras = ["murmur"], version = "^5.8.1" } dictdiffer = "^0.9.0" pathlib = "^1.0.1" platformdirs = "^2.5.4"