From e59fd63237851750c0f5b59be4818d8f1df7179a Mon Sep 17 00:00:00 2001 From: Yarne Coppens Date: Wed, 14 Aug 2024 10:54:11 +0200 Subject: [PATCH] Fixed bug when retrieving plays. BGG was queried too much --- src/modules/bgg_connection.py | 7 ++++--- src/modules/data_connection.py | 15 ++++----------- src/modules/db_connection.py | 14 ++++++++++---- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/modules/bgg_connection.py b/src/modules/bgg_connection.py index 191f99d..df59f62 100644 --- a/src/modules/bgg_connection.py +++ b/src/modules/bgg_connection.py @@ -15,6 +15,7 @@ from src.config import definitions authenticated_session: requests.Session = requests.Session() def url_to_xml_object(url: HttpUrl) -> ET.Element: + r = authenticated_session.get(url) while r.status_code == 202 or r.status_code == 429: @@ -29,7 +30,7 @@ def url_to_xml_object(url: HttpUrl) -> ET.Element: root = ET.fromstring(r.content) return root -def get_boardgame(boardgame_id: int) -> boardgame_classes.BoardGame: +def get_boardgame(boardgame_id: int) -> Union[boardgame_classes.BoardGame, boardgame_classes.BoardGameExpansion]: url : str = "https://boardgamegeek.com/xmlapi2/thing?id={}&stats=true".format(boardgame_id) boardgame_xml_object : ET.Element = url_to_xml_object(url) @@ -62,7 +63,7 @@ def get_multiple_boardgames(boardgame_ids: list[int]) -> list[boardgame_classes. #Requires single boardgame XML 'item' from bgg api on /thing -def convert_xml_to_boardgame(boardgame_xml: ET.Element) -> boardgame_classes.BoardGame: +def convert_xml_to_boardgame(boardgame_xml: ET.Element) -> Union[boardgame_classes.BoardGame, boardgame_classes.BoardGameExpansion]: boardgame_type = boardgame_xml.get('type') @@ -291,7 +292,7 @@ def get_plays() -> list[play_classes.Play]: all_plays : list[play_classes.Play] = [] for page in range(amount_of_pages_needed): - url = 'https://boardgamegeek.com/xmlapi2/plays?username={}&page={}'.format(auth_manager.username, page) + url = 'https://boardgamegeek.com/xmlapi2/plays?username={}&page={}'.format(auth_manager.username, page + 1) plays_page_xml_object = url_to_xml_object(url) for play_xml in plays_page_xml_object: new_play = convert_play_xml_to_play(play_xml) diff --git a/src/modules/data_connection.py b/src/modules/data_connection.py index faa1db7..3df89a4 100644 --- a/src/modules/data_connection.py +++ b/src/modules/data_connection.py @@ -7,7 +7,7 @@ from src.classes import boardgame_classes, play_classes def get_db_session(): return db_connection.get_session() -def get_boardgame(session: Session, boardgame_id: int) -> boardgame_classes.BoardGame: +def get_boardgame(session: Session, boardgame_id: int) -> Union[boardgame_classes.BoardGame, boardgame_classes.BoardGameExpansion]: #Will check if it already exists in db, then it will get it from there boardgame_in_db = db_connection.get_boardgame(session, boardgame_id=boardgame_id) @@ -24,22 +24,15 @@ def get_boardgame(session: Session, boardgame_id: int) -> boardgame_classes.Boar return to_return_boardgame -def get_multiple_boardgames(session: Session, boardgame_ids: list[int]) -> list[boardgame_classes.BoardGame]: +def get_multiple_boardgames(session: Session, boardgame_ids: list[int]) -> list[Union[boardgame_classes.BoardGame, boardgame_classes.BoardGameExpansion]]: - boardgames_in_db, boardgame_ids_missing = db_connection.get_multiple_boardgames(session, boardgame_classes.BoardGame, boardgame_ids=boardgame_ids) - boardgame_expansions_in_db, boardgame_expansion_ids_missing = db_connection.get_multiple_boardgames(session, boardgame_classes.BoardGameExpansion, boardgame_ids=boardgame_ids) - - boardgames_in_db += boardgame_expansions_in_db - boardgame_ids_missing += boardgame_expansion_ids_missing + boardgames_in_db, boardgame_ids_missing = db_connection.get_multiple_boardgames(session, boardgame_ids=boardgame_ids) if len(boardgame_ids_missing) != 0: missing_boardgames = bgg_connection.get_multiple_boardgames(boardgame_ids_missing) db_connection.add_multiple_boardgames(session, missing_boardgames) - boardgames_in_db, boardgame_ids_missing = db_connection.get_multiple_boardgames(session, boardgame_classes.BoardGame, boardgame_ids=boardgame_ids) - boardgame_expansions_in_db, boardgame_expansion_ids_missing = db_connection.get_multiple_boardgames(session, boardgame_classes.BoardGameExpansion, boardgame_ids=boardgame_ids) - - boardgames_in_db += boardgame_expansions_in_db + boardgames_in_db, boardgame_ids_missing = db_connection.get_multiple_boardgames(session, boardgame_ids=boardgame_ids) return boardgames_in_db diff --git a/src/modules/db_connection.py b/src/modules/db_connection.py index d07690a..838e084 100644 --- a/src/modules/db_connection.py +++ b/src/modules/db_connection.py @@ -8,7 +8,7 @@ sqlite_url = definitions.SQLITE_URL connect_args = {"check_same_thread": False} -engine = create_engine(sqlite_url, echo=True, connect_args=connect_args) +engine = create_engine(sqlite_url, echo=False, connect_args=connect_args) def get_session(): with Session(engine) as session: @@ -63,15 +63,21 @@ def get_boardgame(session: Session, boardgame_id: int) -> Union[ return boardgame -def get_multiple_boardgames(session: Session, boardgame_type: SQLModel, boardgame_ids: list[int]) -> tuple[Union[ +def get_multiple_boardgames(session: Session, boardgame_ids: list[int]) -> tuple[Union[ list[boardgame_classes.BoardGame], list[boardgame_classes.BoardGameExpansion]], list[int]]: - - statement = select(boardgame_type).where(boardgame_type.id.in_(boardgame_ids)) + statement = select(boardgame_classes.BoardGame).where(boardgame_classes.BoardGame.id.in_(boardgame_ids)) results = session.exec(statement) boardgames = results.all() + statement = select(boardgame_classes.BoardGameExpansion).where(boardgame_classes.BoardGameExpansion.id.in_(boardgame_ids)) + results = session.exec(statement) + + expansions = results.all() + + boardgames += expansions + missing_boardgame_ids = list(filter(lambda x: x not in [boardgame.id for boardgame in boardgames], boardgame_ids)) return boardgames, missing_boardgame_ids