| @@ -13,17 +13,18 @@ from dataclasses import dataclass | |||||
| from datetime import date, datetime, timedelta | from datetime import date, datetime, timedelta | ||||
| from typing import Any, Type, TypedDict, cast | from typing import Any, Type, TypedDict, cast | ||||
| import mysql.connector | |||||
| import requests | import requests | ||||
| from mysql.connector.connection import MySQLConnectionAbstract | |||||
| from eloquent import DatabaseManager, Model | |||||
| from eloquent.orm.relations.dynamic_property import DynamicProperty | |||||
| class DbNull: | |||||
| def __new__ ( | |||||
| cls, | |||||
| ): | |||||
| delattr (cls, '__init__') | |||||
| DbNullType = Type[DbNull] | |||||
| config: dict[str, DbConfig] = { 'mysql': { 'driver': 'mysql', | |||||
| 'host': 'localhost', | |||||
| 'database': 'nizika_nico', | |||||
| 'user': os.environ['MYSQL_USER'], | |||||
| 'password': os.environ['MYSQL_PASS'], | |||||
| 'prefix': '' } } | |||||
| db = DatabaseManager (config) | |||||
| Model.set_connection_resolver (db) | |||||
| class VideoSearchParam (TypedDict): | class VideoSearchParam (TypedDict): | ||||
| @@ -105,51 +106,165 @@ class VideoTagRow (TypedDict): | |||||
| untagged_at: date | None | untagged_at: date | None | ||||
| def main ( | |||||
| ) -> None: | |||||
| conn = mysql.connector.connect (user = os.environ['MYSQL_USER'], | |||||
| password = os.environ['MYSQL_PASS'], | |||||
| database = 'nizika_nico') | |||||
| if not isinstance (conn, MySQLConnectionAbstract): | |||||
| raise TypeError | |||||
| now = datetime.now () | |||||
| class DbConfig (TypedDict): | |||||
| driver: str | |||||
| host: str | |||||
| database: str | |||||
| user: str | |||||
| password: str | |||||
| prefix: str | |||||
| video_dao = VideoDao (conn) | |||||
| tag_dao = TagDao (conn) | |||||
| video_tag_dao = VideoTagDao (conn) | |||||
| video_history_dao = VideoHistoryDao (conn) | |||||
| comment_dao = CommentDao (conn) | |||||
| user_dao = UserDao (conn) | |||||
| def main ( | |||||
| ) -> None: | |||||
| api_data = search_nico_by_tags (['伊地知ニジカ', 'ぼざろクリーチャーシリーズ']) | api_data = search_nico_by_tags (['伊地知ニジカ', 'ぼざろクリーチャーシリーズ']) | ||||
| update_tables (video_dao, tag_dao, video_tag_dao, video_history_dao, comment_dao, user_dao, | |||||
| api_data, now) | |||||
| update_tables (api_data, now) | |||||
| conn.commit () | conn.commit () | ||||
| conn.close () | conn.close () | ||||
| class Comment (Model): | |||||
| __timestamps__ = False | |||||
| @property | |||||
| def video ( | |||||
| self, | |||||
| ) -> DynamicProperty: | |||||
| return self.belongs_to (Video) | |||||
| @property | |||||
| def user ( | |||||
| self, | |||||
| ) -> DynamicProperty: | |||||
| return self.belongs_to (User) | |||||
| class Tag (Model): | |||||
| __timestamps__ = False | |||||
| @property | |||||
| def video_tags ( | |||||
| self, | |||||
| ) -> DynamicProperty: | |||||
| return self.has_many (VideoTag) | |||||
| @property | |||||
| def videos ( | |||||
| self, | |||||
| ) -> DynamicProperty: | |||||
| return self.belongs_to_many (Video) | |||||
| class User (Model): | |||||
| __timestamps__ = False | |||||
| @property | |||||
| del comments ( | |||||
| self, | |||||
| ) -> DynamicProperty: | |||||
| return self.has_many (Comment) | |||||
| class Video (Model): | |||||
| __timestamps__ = False | |||||
| @property | |||||
| def video_histories ( | |||||
| self, | |||||
| ) -> DynamicProperty: | |||||
| return self.has_many (VideoHistory) | |||||
| @property | |||||
| def video_tags ( | |||||
| self, | |||||
| ) -> DynamicProperty: | |||||
| return self.has_many (VideoTag) | |||||
| @property | |||||
| def tags ( | |||||
| self, | |||||
| ) -> DynamicProperty: | |||||
| return self.belongs_to_many (Tag) | |||||
| @property | |||||
| def comments ( | |||||
| self, | |||||
| ) -> DynamicProperty: | |||||
| return self.has_many (Comment) | |||||
| def upsert ( | |||||
| self, | |||||
| ) -> None: | |||||
| row = Video.where ('code', self.code).first () | |||||
| if row is not None: | |||||
| self.id = row.id | |||||
| self.save () | |||||
| class VideoHistory (Model): | |||||
| __timestamps__ = False | |||||
| @property | |||||
| def video ( | |||||
| self, | |||||
| ) -> DynamicProperty: | |||||
| return self.belongs_to (Video) | |||||
| def upsert ( | |||||
| self, | |||||
| ) -> None: | |||||
| row = (Video | |||||
| .where ('video_id', self.video_id) | |||||
| .where ('fetched_at', self.fetched_at) | |||||
| .first ()) | |||||
| if row is not None: | |||||
| self.id = row.id | |||||
| self.save () | |||||
| class VideoTag (Model): | |||||
| __timestamps__ = False | |||||
| @property | |||||
| def video ( | |||||
| self, | |||||
| ) -> DynamicProperty: | |||||
| return self.belongs_to (Video) | |||||
| @property | |||||
| def tag ( | |||||
| self, | |||||
| ) -> DynamicProperty: | |||||
| return self.belongs_to (Tag) | |||||
| def upsert ( | |||||
| self, | |||||
| ) -> None: | |||||
| row = (Video | |||||
| .where ('video_id', self.video_id) | |||||
| .where ('tag_id', self.tag_id) | |||||
| .first ()) | |||||
| if row is not None: | |||||
| self.id = row.id | |||||
| self.save () | |||||
| def update_tables ( | def update_tables ( | ||||
| video_dao: VideoDao, | |||||
| tag_dao: TagDao, | |||||
| video_tag_dao: VideoTagDao, | |||||
| video_history_dao: VideoHistoryDao, | |||||
| comment_dao: CommentDao, | |||||
| user_dao: UserDao, | |||||
| api_data: list[VideoResult], | |||||
| now: datetime, | |||||
| api_data: list[VideoResult], | |||||
| now: datetime, | |||||
| ) -> None: | ) -> None: | ||||
| video_ids: list[int] = [] | video_ids: list[int] = [] | ||||
| for datum in api_data: | for datum in api_data: | ||||
| tag_names: list[str] = datum['tags'].split () | tag_names: list[str] = datum['tags'].split () | ||||
| video = VideoDto (code = datum['contentId'], | |||||
| title = datum['title'], | |||||
| description = datum['description'] or '', | |||||
| uploaded_at = datetime.fromisoformat (datum['startTime'])) | |||||
| video_dao.upsert (video, False) | |||||
| if video.id_ is not None: | |||||
| video = Video () | |||||
| video.code = datum['contentId'], | |||||
| video.title = datum['title'], | |||||
| video.description = datum['description'] or '', | |||||
| video.uploaded_at = datetime.fromisoformat (datum['startTime']) | |||||
| video.upsert () | |||||
| if video.id is not None: | |||||
| video_ids.append (video.id_) | video_ids.append (video.id_) | ||||
| video_history = VideoHistoryDto (video_id = video.id_, | video_history = VideoHistoryDto (video_id = video.id_, | ||||
| fetched_at = now, | fetched_at = now, | ||||
| @@ -298,12 +413,6 @@ def search_nico_by_tags ( | |||||
| class VideoDao: | class VideoDao: | ||||
| def __init__ ( | |||||
| self, | |||||
| conn: MySQLConnectionAbstract, | |||||
| ): | |||||
| self.conn = conn | |||||
| def find ( | def find ( | ||||
| self, | self, | ||||
| video_id: int, | video_id: int, | ||||
| @@ -473,26 +582,7 @@ class VideoDao: | |||||
| return video | return video | ||||
| @dataclass (slots = True) | |||||
| class VideoDto: | |||||
| code: str | |||||
| title: str | |||||
| description: str | |||||
| uploaded_at: datetime | |||||
| id_: int | None = None | |||||
| deleted_at: datetime | DbNullType = DbNull | |||||
| video_tags: list[VideoTagDto] | None = None | |||||
| comments: list[CommentDto] | None = None | |||||
| video_histories: list[VideoHistoryDto] | None = None | |||||
| class VideoTagDao: | class VideoTagDao: | ||||
| def __init__ ( | |||||
| self, | |||||
| conn: MySQLConnectionAbstract, | |||||
| ): | |||||
| self.conn = conn | |||||
| def fetch_by_video_id ( | def fetch_by_video_id ( | ||||
| self, | self, | ||||
| video_id: int, | video_id: int, | ||||
| @@ -690,24 +780,7 @@ class VideoTagDao: | |||||
| return video_tag | return video_tag | ||||
| @dataclass (slots = True) | |||||
| class VideoTagDto: | |||||
| video_id: int | |||||
| tag_id: int | |||||
| tagged_at: date | |||||
| id_: int | None = None | |||||
| untagged_at: date | DbNullType = DbNull | |||||
| video: VideoDto | None = None | |||||
| tag: TagDto | None = None | |||||
| class TagDao: | class TagDao: | ||||
| def __init__ ( | |||||
| self, | |||||
| conn: MySQLConnectionAbstract, | |||||
| ): | |||||
| self.conn = conn | |||||
| def find ( | def find ( | ||||
| self, | self, | ||||
| tag_id: int, | tag_id: int, | ||||
| @@ -783,19 +856,7 @@ class TagDao: | |||||
| name = row['name']) | name = row['name']) | ||||
| @dataclass (slots = True) | |||||
| class TagDto: | |||||
| name: str | |||||
| id_: int | None = None | |||||
| class VideoHistoryDao: | class VideoHistoryDao: | ||||
| def __init__ ( | |||||
| self, | |||||
| conn: MySQLConnectionAbstract, | |||||
| ): | |||||
| self.conn = conn | |||||
| def fetch_by_video_id ( | def fetch_by_video_id ( | ||||
| self, | self, | ||||
| video_id: int, | video_id: int, | ||||
| @@ -884,22 +945,7 @@ class VideoHistoryDao: | |||||
| return video_history | return video_history | ||||
| @dataclass (slots = True) | |||||
| class VideoHistoryDto: | |||||
| video_id: int | |||||
| fetched_at: date | |||||
| views_count: int | |||||
| id_: int | None = None | |||||
| video: VideoDto | None = None | |||||
| class CommentDao: | class CommentDao: | ||||
| def __init__ ( | |||||
| self, | |||||
| conn: MySQLConnectionAbstract, | |||||
| ): | |||||
| self.conn = conn | |||||
| def fetch_by_video_id ( | def fetch_by_video_id ( | ||||
| self, | self, | ||||
| video_id: int, | video_id: int, | ||||
| @@ -1001,27 +1047,7 @@ class CommentDao: | |||||
| return comment | return comment | ||||
| @dataclass (slots = True) | |||||
| class CommentDto: | |||||
| video_id: int | |||||
| comment_no: int | |||||
| user_id: int | |||||
| content: str | |||||
| posted_at: datetime | |||||
| id_: int | None = None | |||||
| nico_count: int = 0 | |||||
| vpos_ms: int | DbNullType = DbNull | |||||
| video: VideoDto | None = None | |||||
| user: UserDto | None = None | |||||
| class UserDao: | class UserDao: | ||||
| def __init__ ( | |||||
| self, | |||||
| conn: MySQLConnectionAbstract, | |||||
| ): | |||||
| self.conn = conn | |||||
| def fetch_by_code ( | def fetch_by_code ( | ||||
| self, | self, | ||||
| user_code: str | user_code: str | ||||
| @@ -1062,11 +1088,5 @@ class UserDao: | |||||
| code = row['code']) | code = row['code']) | ||||
| @dataclass (slots = True) | |||||
| class UserDto: | |||||
| code: str | |||||
| id_: int | None = None | |||||
| if __name__ == '__main__': | if __name__ == '__main__': | ||||
| main () | main () | ||||