From e4b541ecdc6c597a42ffcc9ff616e0f9b9f37a06 Mon Sep 17 00:00:00 2001 From: miteruzo Date: Mon, 7 Oct 2024 22:53:13 +0900 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E6=88=90=EF=BC=88=E3=81=9F=E3=81=B6?= =?UTF-8?q?=E3=82=93=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- update_db.py | 158 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 137 insertions(+), 21 deletions(-) diff --git a/update_db.py b/update_db.py index cf96f56..6da1bb4 100644 --- a/update_db.py +++ b/update_db.py @@ -1,3 +1,7 @@ +""" +日次で実行し，ぼざクリ DB を最新に更新する． +""" + from __future__ import annotations import json @@ -7,7 +11,7 @@ import string import time from dataclasses import dataclass from datetime import datetime, timedelta -from typing import TypedDict, cast +from typing import Any, TypedDict, cast import mysql.connector import requests @@ -16,15 +20,6 @@ DbNull = (None,) DbNullType = tuple[None] -class VideoResult (TypedDict): - contentId: str - title: str - tags: list[str] - description: str - viewCounter: int - startTime: str - - class VideoSearchParam (TypedDict): q: str targets: str @@ -34,8 +29,29 @@ class VideoSearchParam (TypedDict): jsonFilter: str +class VideoResult (TypedDict): + contentId: str + title: str + tags: list[str] + description: str + viewCounter: int + startTime: str + + class CommentResult (TypedDict): - pass + id: str + no: int + vposMs: int + body: str + commands: list[str] + userId: str + isPremium: bool + score: int + postedAt: str + nicoruCount: int + nicoruId: Any + source: str + isMyPost: bool def main ( @@ -50,10 +66,13 @@ def main ( tag_dao = TagDao (conn) video_tag_dao = VideoTagDao (conn) video_history_dao = VideoHistoryDao (conn) + comment_dao = CommentDao (conn) + user_dao = UserDao (conn) api_data = search_nico_by_tags (['伊地知ニジカ', 'ぼざろクリーチャーシリーズ']) - update_tables (video_dao, tag_dao, video_tag_dao, video_history_dao, api_data, now) + update_tables (video_dao, tag_dao, video_tag_dao, video_history_dao, comment_dao, user_dao, + api_data, now) # TODO: 書くこと @@ -63,9 +82,12 @@ def update_tables ( tag_dao: TagDao, video_tag_dao: VideoTagDao, video_history_dao: VideoHistoryDao, + comment_dao: CommentDao, + user_dao: UserDao, api_data: list[VideoResult], now: datetime, ) -> None: + video_ids: list[int] = [] for datum in api_data: video = VideoDto (code = datum['contentId'], title = datum['title'], @@ -73,6 +95,7 @@ def update_tables ( uploaded_at = datetime.fromisoformat (datum['startTime'])) video_dao.upsert (video, False) if video.id_ is not None: + video_ids.append (video.id_) video_history = VideoHistoryDto (video_id = video.id_, fetched_at = now, views_count = datum['viewCounter']) @@ -99,17 +122,37 @@ def update_tables ( tag_id = tag.id_, tagged_at = now) video_tag_dao.insert (video_tag, False) - # TODO: コメント取得すること - - # TODO: 削除処理，存在しなぃ動画リストを取得した上で行ふこと - # video_dao.delete (video_ids, now) - - # TODO: 書くこと + for com in fetch_comments (video.code): + user = user_dao.fetch_by_code (com['userId']) + if user is None: + user = UserDto (code = com['userId']) + user_dao.insert (user) + if video.id_ is not None and user.id_ is not None: + comment = CommentDto (video_id = video.id_, + comment_no = com['no'], + user_id = user.id_, + content = com['body'], + posted_at = datetime.fromisoformat (com['postedAt']), + nico_count = com['nicoruCount'], + vpos_ms = com['vposMs']) + comment_dao.upsert (comment, False) + + alive_video_ids = [d['contentId'] for d in api_data] + + lost_video_ids: list[int] = [] + videos = video_dao.fetch_alive () + for video in videos: + if video.id_ is not None and video.id_ not in alive_video_ids: + lost_video_ids.append (video.id_) + + video_dao.delete (lost_video_ids, now) def fetch_comments ( - video_id: str, -) -> list: + video_code: str, +) -> list[CommentResult]: + time.sleep (1.2) + headers = { 'X-Frontend-Id': '6', 'X-Frontend-Version': '0' } @@ -119,7 +162,7 @@ def fetch_comments ( + '_' + str (random.randrange (10 ** 12, 10 ** 13))) - url = (f"https://www.nicovideo.jp/api/watch/v3_guest/{ video_id }" + url = (f"https://www.nicovideo.jp/api/watch/v3_guest/{ video_code }" + f"?actionTrackId={ action_track_id }") res = requests.post (url, headers = headers, timeout = 60).json () @@ -253,6 +296,27 @@ class VideoDao: videos.append (self._create_dto_from_row (row, with_relation_tables)) return videos + def fetch_alive ( + self, + ) -> list[VideoDto]: + with self.conn.cursor () as c: + c.execute (""" + SELECT + id, + code, + title, + description, + uploaded_at, + deleted_at + FROM + videos + WHERE + deleted_at IS NULL""") + videos: list[VideoDto] = [] + for row in c.fetchall (): + videos.append (self._create_dto_from_row (row, False)) + return videos + def upsert ( self, video: VideoDto, @@ -836,6 +900,58 @@ class CommentDto: nico_count: int = 0 vpos_ms: int | DbNullType = DbNull video: VideoDto | None = None + user: UserDto | None = None + + +class UserDao: + def __init__ ( + self, + conn, + ): + self.conn = conn + + def fetch_by_code ( + self, + user_code: str + ) -> UserDto | None: + with self.conn.cursor () as c: + c.execute (""" + SELECT + id, + code + FROM + users + WHERE + code = %s""", user_code) + row = c.fetchone () + if row is None: + return None + return self._create_dto_from_row (row) + + def insert ( + self, + user: UserDto, + ) -> None: + with self.conn.cursor () as c: + c.execute (""" + INSERT INTO + users(code) + VALUES + (%s)""", user.code) + user.id_ = c.lastrowid + + def _create_dto_from_row ( + self, + row, + ) -> UserDto: + return UserDto (id_ = row['id'], + code = row['code']) + + +@dataclass (slots = True) +class UserDto: + code: str + id_: int | None = None if __name__ == '__main__':