DAO および DTO の不足について対応
このコミットが含まれているのは:
+491
-39
@@ -33,6 +33,10 @@ class VideoSearchParam (TypedDict):
|
||||
jsonFilter: str
|
||||
|
||||
|
||||
class CommentResult (TypedDict):
|
||||
pass
|
||||
|
||||
|
||||
def main (
|
||||
) -> None:
|
||||
conn = mysql.connector.connect (host = os.environ['MYSQL_HOST'],
|
||||
@@ -43,38 +47,61 @@ def main (
|
||||
|
||||
video_dao = VideoDao (conn)
|
||||
tag_dao = TagDao (conn)
|
||||
video_tag_dao = VideoTagDao (conn)
|
||||
video_history_dao = VideoHistoryDao (conn)
|
||||
|
||||
api_data = search_nico_by_tags (['伊地知ニジカ', 'ぼざろクリーチャーシリーズ'])
|
||||
|
||||
update_video_and_tag_table (video_dao, tag_dao, api_data, now)
|
||||
update_tables (video_dao, tag_dao, video_tag_dao, video_history_dao, api_data, now)
|
||||
|
||||
# TODO: 書くこと
|
||||
|
||||
|
||||
def update_video_and_tag_table (
|
||||
video_dao: VideoDao,
|
||||
tag_dao: TagDao,
|
||||
api_data: list[VideoResult],
|
||||
now: datetime,
|
||||
def update_tables (
|
||||
video_dao: VideoDao,
|
||||
tag_dao: TagDao,
|
||||
video_tag_dao: VideoTagDao,
|
||||
video_history_dao: VideoHistoryDao,
|
||||
api_data: list[VideoResult],
|
||||
now: datetime,
|
||||
) -> None:
|
||||
videos: list[VideoDto] = []
|
||||
for datum in api_data:
|
||||
tags: list[TagDto] = []
|
||||
for tag_name in datum['tags']:
|
||||
tags.append (TagDto (name = tag_name))
|
||||
tag_dao.upsert_all (tags)
|
||||
# TODO: タグの対応づけすること
|
||||
# TODO: コメント取得すること
|
||||
video = VideoDto (code = datum['contentId'],
|
||||
title = datum['title'],
|
||||
description = datum['description'],
|
||||
uploaded_at = datum['startTime'],
|
||||
deleted_at = DbNull)
|
||||
videos.append (video)
|
||||
uploaded_at = datum['startTime'])
|
||||
video_dao.upsert (video, False)
|
||||
if video.id_ is not None:
|
||||
video_history = VideoHistoryDto (video_id = video.id_,
|
||||
fetched_at = now,
|
||||
views_count = datum['viewCounter'])
|
||||
video_history_dao.insert (video_history)
|
||||
tag_ids: list[int] = []
|
||||
video_tags = video_tag_dao.fetch_alive_by_video_id (video.id_, False)
|
||||
for vt in video_tags:
|
||||
tag = tag_dao.find (vt.tag_id)
|
||||
if (tag is not None
|
||||
and (tag.name not in datum['tags'])
|
||||
and (tag.id_ is not None)):
|
||||
tag_ids.append (tag.id_)
|
||||
video_tag_dao.untag_all (video.id_, tag_ids, now)
|
||||
tags: list[TagDto] = []
|
||||
for tag_name in datum['tags']:
|
||||
tag = tag_dao.fetch_by_name (tag_name)
|
||||
if tag is None:
|
||||
tag = TagDto (name = tag_name)
|
||||
tag_dao.insert (tag)
|
||||
if video.id_ is not None and tag.id_ is not None:
|
||||
video_tag = video_tag_dao.fetch_alive_by_ids (video.id_, tag.id_, False)
|
||||
if video_tag is None:
|
||||
video_tag = VideoTagDto (video_id = video.id_,
|
||||
tag_id = tag.id_,
|
||||
tagged_at = now)
|
||||
video_tag_dao.insert (video_tag, False)
|
||||
# TODO: コメント取得すること
|
||||
|
||||
video_dao.upsert_all (videos)
|
||||
|
||||
video_dao.delete_nonexistent_data (video_id_list)
|
||||
# TODO: 削除処理，存在しなぃ動画リストを取得した上で行ふこと
|
||||
# video_dao.delete (video_ids, now)
|
||||
|
||||
# TODO: 書くこと
|
||||
|
||||
@@ -87,7 +114,7 @@ def fetch_comments (
|
||||
|
||||
action_track_id = (
|
||||
''.join (random.choice (string.ascii_letters + string.digits)
|
||||
for _ in range (10))
|
||||
for _ in range (10))
|
||||
+ '_'
|
||||
+ str (random.randrange (10 ** 12, 10 ** 13)))
|
||||
|
||||
@@ -139,7 +166,7 @@ def search_nico_by_tags (
|
||||
# TODO: 年月日の設定ができてゐなぃのと，100 件までしか取得できなぃので何とかすること
|
||||
|
||||
query_filter = json.dumps ({ 'type': 'or',
|
||||
'filters': [
|
||||
'filters': [
|
||||
{ 'type': 'range',
|
||||
'field': 'startTime',
|
||||
'from': f"{year}-{start}T00:00:00+09:00",
|
||||
@@ -166,10 +193,10 @@ class VideoDao:
|
||||
):
|
||||
self.conn = conn
|
||||
|
||||
def fetch (
|
||||
def find (
|
||||
self,
|
||||
video_id: int,
|
||||
with_relation_tables: bool = True,
|
||||
with_relation_tables: bool,
|
||||
) -> VideoDto | None:
|
||||
with self.conn.cursor () as c:
|
||||
c.execute ("""
|
||||
@@ -193,7 +220,7 @@ class VideoDao:
|
||||
|
||||
def fetch_all (
|
||||
self,
|
||||
with_relation_tables: bool = True,
|
||||
with_relation_tables: bool,
|
||||
) -> list[VideoDto]:
|
||||
with self.conn.cursor () as c:
|
||||
c.execute ("""
|
||||
@@ -216,7 +243,7 @@ class VideoDao:
|
||||
def upsert (
|
||||
self,
|
||||
video: VideoDto,
|
||||
with_relation_tables: bool = True,
|
||||
with_relation_tables: bool,
|
||||
) -> None:
|
||||
with self.conn.cursor () as c:
|
||||
c.execute ("""
|
||||
@@ -246,18 +273,34 @@ class VideoDao:
|
||||
video.deleted_at))
|
||||
video.id_ = c.lastrowid
|
||||
if with_relation_tables:
|
||||
VideoTagDao (self.conn).upsert_all (video.video_tags, False)
|
||||
CommentDao (self.conn).upsert_all (video.comments, False)
|
||||
VideoHistoryDao (self.conn).upsert_all (video.video_histories, False)
|
||||
if video.video_tags is not None:
|
||||
VideoTagDao (self.conn).upsert_all (video.video_tags, False)
|
||||
if video.comments is not None:
|
||||
CommentDao (self.conn).upsert_all (video.comments, False)
|
||||
if video.video_histories is not None:
|
||||
VideoHistoryDao (self.conn).upsert_all (video.video_histories)
|
||||
|
||||
def upsert_all (
|
||||
self,
|
||||
videos: list[VideoDto],
|
||||
with_relation_tables: bool = True,
|
||||
with_relation_tables: bool,
|
||||
) -> None:
|
||||
for video in videos:
|
||||
self.upsert (video, with_relation_tables)
|
||||
|
||||
def delete (
|
||||
self,
|
||||
video_ids: list[int],
|
||||
at: datetime,
|
||||
) -> None:
|
||||
with self.conn.cursor () as c:
|
||||
for video in videos:
|
||||
self.upsert (video, with_relation_tables)
|
||||
c.execute ("""
|
||||
UPDATE
|
||||
videos
|
||||
SET
|
||||
deleted_at = %s
|
||||
WHERE
|
||||
id IN (%s)""", (at, (*video_ids,)))
|
||||
|
||||
def _create_dto_from_row (
|
||||
self,
|
||||
@@ -270,8 +313,8 @@ class VideoDao:
|
||||
description = row['description'],
|
||||
uploaded_at = row['uploaded_at'],
|
||||
deleted_at = row['deleted_at'])
|
||||
if with_relation_tables:
|
||||
video.video_tags = VideoTagDao (self.conn).fetch_by_video_id (cast (int, video.id_), False)
|
||||
if with_relation_tables and video.id_ is not None:
|
||||
video.video_tags = VideoTagDao (self.conn).fetch_by_video_id (video.id_, False)
|
||||
for i in range (len (video.video_tags)):
|
||||
video.video_tags[i].video = video
|
||||
video.comments = CommentDao (self.conn).fetch_by_video_id (video.id_, False)
|
||||
@@ -306,7 +349,7 @@ class VideoTagDao:
|
||||
def fetch_by_video_id (
|
||||
self,
|
||||
video_id: int,
|
||||
with_relation_tables: bool = True,
|
||||
with_relation_tables: bool,
|
||||
) -> list[VideoTagDto]:
|
||||
with self.conn.cursor () as c:
|
||||
c.execute ("""
|
||||
@@ -327,6 +370,140 @@ class VideoTagDao:
|
||||
video_tags.append (self._create_dto_from_row (row, with_relation_tables))
|
||||
return video_tags
|
||||
|
||||
def fetch_alive_by_video_id (
|
||||
self,
|
||||
video_id: int,
|
||||
with_relation_tables: bool,
|
||||
) -> list[VideoTagDto]:
|
||||
with self.conn.cursor () as c:
|
||||
c.execute ("""
|
||||
SELECT
|
||||
id,
|
||||
video_id,
|
||||
tag_id,
|
||||
tagged_at,
|
||||
untagged_at
|
||||
FROM
|
||||
video_tags
|
||||
WHERE
|
||||
video_id = %s
|
||||
AND (untagged_at IS NULL)
|
||||
ORDER BY
|
||||
id""", video_id)
|
||||
video_tags: list[VideoTagDto] = []
|
||||
for row in c.fetchall ():
|
||||
video_tags.append (self._create_dto_from_row (row, with_relation_tables))
|
||||
return video_tags
|
||||
|
||||
def fetch_alive_by_ids (
|
||||
self,
|
||||
video_id: int,
|
||||
tag_id: int,
|
||||
with_relation_tables: bool,
|
||||
) -> VideoTagDto | None:
|
||||
with self.conn.cursor () as c:
|
||||
c.execute ("""
|
||||
SELECT
|
||||
id,
|
||||
video_id,
|
||||
tag_id,
|
||||
tagged_at,
|
||||
untagged_at
|
||||
FROM
|
||||
video_tags
|
||||
WHERE
|
||||
video_id = %s
|
||||
AND tag_id = %s""", (video_id, tag_id))
|
||||
row = c.fetchone ()
|
||||
if row is None:
|
||||
return None
|
||||
return self._create_dto_from_row (row, with_relation_tables)
|
||||
|
||||
def insert (
|
||||
self,
|
||||
video_tag: VideoTagDto,
|
||||
with_relation_tables: bool,
|
||||
) -> None:
|
||||
with self.conn.cursor () as c:
|
||||
c.execute ("""
|
||||
INSERT INTO
|
||||
video_tags(
|
||||
video_id,
|
||||
tag_id,
|
||||
tagged_at,
|
||||
untagged_at)
|
||||
VALUES
|
||||
(
|
||||
%s,
|
||||
%s,
|
||||
%s,
|
||||
%s)""", (video_tag.video_id, video_tag.tag_id,
|
||||
video_tag.tagged_at, video_tag.untagged_at))
|
||||
video_tag.id_ = c.lastrowid
|
||||
if with_relation_tables:
|
||||
if video_tag.video is not None:
|
||||
VideoDao (self.conn).upsert (video_tag.video, True)
|
||||
if video_tag.tag is not None:
|
||||
TagDao (self.conn).upsert (video_tag.tag)
|
||||
|
||||
def upsert (
|
||||
self,
|
||||
video_tag: VideoTagDto,
|
||||
with_relation_tables: bool,
|
||||
) -> None:
|
||||
with self.conn.cursor () as c:
|
||||
c.execute ("""
|
||||
INSERT INTO
|
||||
video_tags(
|
||||
video_id,
|
||||
tag_id,
|
||||
tagged_at,
|
||||
untagged_at)
|
||||
VALUES
|
||||
(
|
||||
%s,
|
||||
%s,
|
||||
%s,
|
||||
%s)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
video_id = VALUES(video_id),
|
||||
tag_id = VALUES(tag_id),
|
||||
tagged_at = VALUES(tagged_at),
|
||||
untagged_at = VALUES(untagged_at)""", (video_tag.video_id,
|
||||
video_tag.tag_id,
|
||||
video_tag.tagged_at,
|
||||
video_tag.untagged_at))
|
||||
video_tag.id_ = c.lastrowid
|
||||
if with_relation_tables:
|
||||
if video_tag.video is not None:
|
||||
VideoDao (self.conn).upsert (video_tag.video, True)
|
||||
if video_tag.tag is not None:
|
||||
TagDao (self.conn).upsert (video_tag.tag)
|
||||
|
||||
def upsert_all (
|
||||
self,
|
||||
video_tags: list[VideoTagDto],
|
||||
with_relation_tables: bool,
|
||||
) -> None:
|
||||
for video_tag in video_tags:
|
||||
self.upsert (video_tag, with_relation_tables)
|
||||
|
||||
def untag_all (
|
||||
self,
|
||||
video_id: int,
|
||||
tag_ids: list[int],
|
||||
now: datetime
|
||||
) -> None:
|
||||
with self.conn.cursor () as c:
|
||||
c.execute ("""
|
||||
UPDATE
|
||||
video_tags
|
||||
SET
|
||||
untagged_at = %s
|
||||
WHERE
|
||||
video_id = %s
|
||||
AND tag_ids IN (%s)""", (now, video_id, (*tag_ids,)))
|
||||
|
||||
def _create_dto_from_row (
|
||||
self,
|
||||
row,
|
||||
@@ -338,28 +515,303 @@ class VideoTagDao:
|
||||
tagged_at = row['tagged_at'],
|
||||
untagged_at = row['untagged_at'])
|
||||
if with_relation_tables:
|
||||
video_tag.video = VideoDao (self.conn).fetch (video_tag.video_id, True)
|
||||
video_tag.tag = TagDao (self.conn).fetch (video_tag.tag_id, True)
|
||||
video_tag.video = VideoDao (self.conn).find (video_tag.video_id, True)
|
||||
video_tag.tag = TagDao (self.conn).find (video_tag.tag_id)
|
||||
return video_tag
|
||||
|
||||
|
||||
@dataclass (slots = True)
|
||||
class VideoTagDto:
|
||||
video_id: int
|
||||
tag_id: int
|
||||
tagged_at: datetime
|
||||
id_: int | None = None
|
||||
video_id: int | None = None
|
||||
tag_id: int | None = None
|
||||
untagged_at: datetime | DbNullType = DbNull
|
||||
video: VideoDto | None = None
|
||||
tag: TagDto | None = None
|
||||
|
||||
|
||||
class TagDao:
|
||||
def __init__ (
|
||||
self,
|
||||
conn,
|
||||
):
|
||||
self.conn = conn
|
||||
|
||||
def find (
|
||||
self,
|
||||
tag_id: int,
|
||||
) -> TagDto | None:
|
||||
with self.conn.cursor () as c:
|
||||
c.execute ("""
|
||||
SELECT
|
||||
id,
|
||||
name)
|
||||
FROM
|
||||
tags
|
||||
WHERE
|
||||
id = %s""", tag_id)
|
||||
row = c.fetchone ()
|
||||
if row is None:
|
||||
return None
|
||||
return self._create_dto_from_row (row)
|
||||
|
||||
def fetch_by_name (
|
||||
self,
|
||||
tag_name: str,
|
||||
) -> TagDto | None:
|
||||
with self.conn.cursor () as c:
|
||||
c.execute ("""
|
||||
SELECT
|
||||
id,
|
||||
name
|
||||
FROM
|
||||
tags
|
||||
WHERE
|
||||
name = %s""", tag_name)
|
||||
row = c.fetchone ()
|
||||
if row is None:
|
||||
return None
|
||||
return self._create_dto_from_row (row)
|
||||
|
||||
def insert (
|
||||
self,
|
||||
tag: TagDto,
|
||||
) -> None:
|
||||
with self.conn.cursor () as c:
|
||||
c.execute ("""
|
||||
INSERT INTO
|
||||
tags(name)
|
||||
VALUES
|
||||
(%s)""", tag.name)
|
||||
tag.id_ = c.lastrowid
|
||||
|
||||
def upsert (
|
||||
self,
|
||||
tag: TagDto,
|
||||
) -> None:
|
||||
with self.conn.cursor () as c:
|
||||
c.execute ("""
|
||||
INSERT INTO
|
||||
tags(name)
|
||||
VALUES
|
||||
(%s)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
name = VALUES(name)""", tag.name)
|
||||
tag.id_ = c.lastrowid
|
||||
|
||||
def _create_dto_from_row (
|
||||
self,
|
||||
row,
|
||||
) -> TagDto:
|
||||
return TagDto (id_ = row['id'],
|
||||
name = row['name'])
|
||||
|
||||
|
||||
@dataclass (slots = True)
|
||||
class TagDto:
|
||||
name: str
|
||||
id_: int | None = None
|
||||
|
||||
|
||||
class VideoHistoryDao:
|
||||
def __init__ (
|
||||
self,
|
||||
conn,
|
||||
):
|
||||
self.conn = conn
|
||||
|
||||
def fetch_by_video_id (
|
||||
self,
|
||||
video_id: int,
|
||||
with_relation_tables: bool,
|
||||
) -> list[VideoHistoryDto]:
|
||||
with self.conn.cursor () as c:
|
||||
c.execute ("""
|
||||
SELECT
|
||||
id,
|
||||
video_id,
|
||||
fetched_at,
|
||||
views_count
|
||||
FROM
|
||||
video_histories
|
||||
WHERE
|
||||
video_id = %s""", video_id)
|
||||
video_histories: list[VideoHistoryDto] = []
|
||||
for row in c.fetchall ():
|
||||
video_histories.append (self._create_dto_from_row (row, with_relation_tables))
|
||||
return video_histories
|
||||
|
||||
def insert (
|
||||
self,
|
||||
video_history: VideoHistoryDto,
|
||||
) -> None:
|
||||
with self.conn.cursor () as c:
|
||||
c.execute ("""
|
||||
INSERT INTO
|
||||
video_histories(
|
||||
video_id,
|
||||
fetched_at,
|
||||
views_count)
|
||||
VALUES
|
||||
(
|
||||
%s,
|
||||
%s,
|
||||
%s)""", (video_history.video_id,
|
||||
video_history.fetched_at,
|
||||
video_history.views_count))
|
||||
|
||||
def upsert (
|
||||
self,
|
||||
video_history: VideoHistoryDto,
|
||||
) -> None:
|
||||
with self.conn.cursor () as c:
|
||||
c.execute ("""
|
||||
INSERT INTO
|
||||
video_histories(
|
||||
video_id,
|
||||
fetched_at,
|
||||
views_count)
|
||||
VALUES
|
||||
(
|
||||
%s,
|
||||
%s,
|
||||
%s)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
video_id,
|
||||
fetched_at,
|
||||
views_count""", (video_history.video_id,
|
||||
video_history.fetched_at,
|
||||
video_history.views_count))
|
||||
|
||||
def upsert_all (
|
||||
self,
|
||||
video_histories: list[VideoHistoryDto],
|
||||
) -> None:
|
||||
for video_history in video_histories:
|
||||
self.upsert (video_history)
|
||||
|
||||
def _create_dto_from_row (
|
||||
self,
|
||||
row,
|
||||
with_relation_tables: bool,
|
||||
) -> VideoHistoryDto:
|
||||
video_history = VideoHistoryDto (id_ = row['id'],
|
||||
video_id = row['video_id'],
|
||||
fetched_at = row['fetched_at'],
|
||||
views_count = row['views_count'])
|
||||
if with_relation_tables:
|
||||
video_history.video = VideoDao (self.conn).find (video_history.video_id, True)
|
||||
return video_history
|
||||
|
||||
|
||||
@dataclass (slots = True)
|
||||
class VideoHistoryDto:
|
||||
video_id: int
|
||||
fetched_at: datetime
|
||||
views_count: int
|
||||
id_: int | None = None
|
||||
video: VideoDto | None = None
|
||||
|
||||
|
||||
class CommentDao:
|
||||
def __init__ (
|
||||
self,
|
||||
conn,
|
||||
):
|
||||
self.conn = conn
|
||||
|
||||
def fetch_by_video_id (
|
||||
self,
|
||||
video_id: int,
|
||||
with_relation_tables: bool,
|
||||
) -> list[CommentDto]:
|
||||
with self.conn.cursor () as c:
|
||||
c.execute ("""
|
||||
SELECT
|
||||
id,
|
||||
video_id,
|
||||
comment_no,
|
||||
user_id,
|
||||
content,
|
||||
posted_at,
|
||||
nico_count,
|
||||
vpos_ms
|
||||
FROM
|
||||
comments
|
||||
WHERE
|
||||
video_id = %s""", video_id)
|
||||
comments: list[CommentDto] = []
|
||||
for row in c.fetchall ():
|
||||
comments.append (self._create_dto_from_row (row, with_relation_tables))
|
||||
return comments
|
||||
|
||||
def upsert (
|
||||
self,
|
||||
comment: CommentDto,
|
||||
with_relation_tables: bool,
|
||||
) -> None:
|
||||
with self.conn.cursor () as c:
|
||||
c.execute ("""
|
||||
INSERT INTO
|
||||
comments(
|
||||
video_id,
|
||||
comment_no,
|
||||
user_id,
|
||||
content,
|
||||
posted_at,
|
||||
nico_count,
|
||||
vpos_ms)
|
||||
VALUES
|
||||
(
|
||||
%s,
|
||||
%s,
|
||||
%s,
|
||||
%s,
|
||||
%s,
|
||||
%s,
|
||||
%s)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
video_id = VALUES(video_id),
|
||||
comment_no = VALUES(comment_no),
|
||||
user_id = VALUES(user_id),
|
||||
content = VALUES(content),
|
||||
posted_at = VALUES(posted_at),
|
||||
nico_count = VALUES(nico_count),
|
||||
vpos_ms = VALUES(vpos_ms)""", (comment.video_id,
|
||||
comment.comment_no,
|
||||
comment.user_id,
|
||||
comment.content,
|
||||
comment.posted_at,
|
||||
comment.nico_count,
|
||||
comment.vpos_ms))
|
||||
|
||||
def upsert_all (
|
||||
self,
|
||||
comments: list[CommentDto],
|
||||
with_relation_tables: bool,
|
||||
) -> None:
|
||||
for comment in comments:
|
||||
self.upsert (comment, with_relation_tables)
|
||||
|
||||
def _create_dto_from_row (
|
||||
self,
|
||||
row,
|
||||
with_relation_tables: bool,
|
||||
) -> CommentDto:
|
||||
comment = CommentDto (id_ = row['id'],
|
||||
video_id = row['video_id'],
|
||||
comment_no = row['comment_no'],
|
||||
user_id = row['user_id'],
|
||||
content = row['content'],
|
||||
posted_at = row['posted_at'],
|
||||
nico_count = row['nico_count'],
|
||||
vpos_ms = row['vpos_ms'])
|
||||
if with_relation_tables:
|
||||
comment.video = VideoDao (self.conn).find (comment.video_id, True)
|
||||
return comment
|
||||
|
||||
|
||||
@dataclass (slots = True)
|
||||
class CommentDto:
|
||||
video_id: int
|
||||
@@ -370,7 +822,7 @@ class CommentDto:
|
||||
id_: int | None = None
|
||||
nico_count: int = 0
|
||||
vpos_ms: int | DbNullType = DbNull
|
||||
|
||||
video: VideoDto | None = None
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
新しい課題から参照
ユーザをブロックする