miteruzo
/
nizika_nico
1
0
Fork 0
Code Issues 6 Pull Requests 0 Releases 0 Wiki Activity
Browse Source

DAO および DTO の不足について対応

feature/query
みてるぞ 1 year ago
parent
f7555345e8
commit
7407664264
1 changed files with 491 additions and 39 deletions
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +491
    -39
      update_db.py

+ 491
- 39
update_db.py View File

@@ -33,6 +33,10 @@ class VideoSearchParam (TypedDict):
jsonFilter: str jsonFilter: str




class CommentResult (TypedDict):
pass


def main ( def main (
) -> None: ) -> None:
conn = mysql.connector.connect (host = os.environ['MYSQL_HOST'], conn = mysql.connector.connect (host = os.environ['MYSQL_HOST'],
@@ -43,38 +47,61 @@ def main (


video_dao = VideoDao (conn) video_dao = VideoDao (conn)
tag_dao = TagDao (conn) tag_dao = TagDao (conn)
video_tag_dao = VideoTagDao (conn)
video_history_dao = VideoHistoryDao (conn)


api_data = search_nico_by_tags (['伊地知ニジカ', 'ぼざろクリーチャーシリーズ']) api_data = search_nico_by_tags (['伊地知ニジカ', 'ぼざろクリーチャーシリーズ'])


update_video_and_tag_table (video_dao, tag_dao, api_data, now)
update_tables (video_dao, tag_dao, video_tag_dao, video_history_dao, api_data, now)


# TODO: 書くこと # TODO: 書くこと




def update_video_and_tag_table (
video_dao: VideoDao,
tag_dao: TagDao,
api_data: list[VideoResult],
now: datetime,
def update_tables (
video_dao: VideoDao,
tag_dao: TagDao,
video_tag_dao: VideoTagDao,
video_history_dao: VideoHistoryDao,
api_data: list[VideoResult],
now: datetime,
) -> None: ) -> None:
videos: list[VideoDto] = []
for datum in api_data: for datum in api_data:
tags: list[TagDto] = []
for tag_name in datum['tags']:
tags.append (TagDto (name = tag_name))
tag_dao.upsert_all (tags)
# TODO: タグの対応づけすること
# TODO: コメント取得すること
video = VideoDto (code = datum['contentId'], video = VideoDto (code = datum['contentId'],
title = datum['title'], title = datum['title'],
description = datum['description'], description = datum['description'],
uploaded_at = datum['startTime'],
deleted_at = DbNull)
videos.append (video)

video_dao.upsert_all (videos)
uploaded_at = datum['startTime'])
video_dao.upsert (video, False)
if video.id_ is not None:
video_history = VideoHistoryDto (video_id = video.id_,
fetched_at = now,
views_count = datum['viewCounter'])
video_history_dao.insert (video_history)
tag_ids: list[int] = []
video_tags = video_tag_dao.fetch_alive_by_video_id (video.id_, False)
for vt in video_tags:
tag = tag_dao.find (vt.tag_id)
if (tag is not None
and (tag.name not in datum['tags'])
and (tag.id_ is not None)):
tag_ids.append (tag.id_)
video_tag_dao.untag_all (video.id_, tag_ids, now)
tags: list[TagDto] = []
for tag_name in datum['tags']:
tag = tag_dao.fetch_by_name (tag_name)
if tag is None:
tag = TagDto (name = tag_name)
tag_dao.insert (tag)
if video.id_ is not None and tag.id_ is not None:
video_tag = video_tag_dao.fetch_alive_by_ids (video.id_, tag.id_, False)
if video_tag is None:
video_tag = VideoTagDto (video_id = video.id_,
tag_id = tag.id_,
tagged_at = now)
video_tag_dao.insert (video_tag, False)
# TODO: コメント取得すること


video_dao.delete_nonexistent_data (video_id_list)
# TODO: 削除処理，存在しなぃ動画リストを取得した上で行ふこと
# video_dao.delete (video_ids, now)


# TODO: 書くこと # TODO: 書くこと


@@ -87,7 +114,7 @@ def fetch_comments (


action_track_id = ( action_track_id = (
''.join (random.choice (string.ascii_letters + string.digits) ''.join (random.choice (string.ascii_letters + string.digits)
for _ in range (10))
for _ in range (10))
+ '_' + '_'
+ str (random.randrange (10 ** 12, 10 ** 13))) + str (random.randrange (10 ** 12, 10 ** 13)))


@@ -139,7 +166,7 @@ def search_nico_by_tags (
# TODO: 年月日の設定ができてゐなぃのと，100 件までしか取得できなぃので何とかすること # TODO: 年月日の設定ができてゐなぃのと，100 件までしか取得できなぃので何とかすること


query_filter = json.dumps ({ 'type': 'or', query_filter = json.dumps ({ 'type': 'or',
'filters': [
'filters': [
{ 'type': 'range', { 'type': 'range',
'field': 'startTime', 'field': 'startTime',
'from': f"{year}-{start}T00:00:00+09:00", 'from': f"{year}-{start}T00:00:00+09:00",
@@ -166,10 +193,10 @@ class VideoDao:
): ):
self.conn = conn self.conn = conn


def fetch (
def find (
self, self,
video_id: int, video_id: int,
with_relation_tables: bool = True,
with_relation_tables: bool,
) -> VideoDto | None: ) -> VideoDto | None:
with self.conn.cursor () as c: with self.conn.cursor () as c:
c.execute (""" c.execute ("""
@@ -193,7 +220,7 @@ class VideoDao:


def fetch_all ( def fetch_all (
self, self,
with_relation_tables: bool = True,
with_relation_tables: bool,
) -> list[VideoDto]: ) -> list[VideoDto]:
with self.conn.cursor () as c: with self.conn.cursor () as c:
c.execute (""" c.execute ("""
@@ -216,7 +243,7 @@ class VideoDao:
def upsert ( def upsert (
self, self,
video: VideoDto, video: VideoDto,
with_relation_tables: bool = True,
with_relation_tables: bool,
) -> None: ) -> None:
with self.conn.cursor () as c: with self.conn.cursor () as c:
c.execute (""" c.execute ("""
@@ -246,18 +273,34 @@ class VideoDao:
video.deleted_at)) video.deleted_at))
video.id_ = c.lastrowid video.id_ = c.lastrowid
if with_relation_tables: if with_relation_tables:
VideoTagDao (self.conn).upsert_all (video.video_tags, False)
CommentDao (self.conn).upsert_all (video.comments, False)
VideoHistoryDao (self.conn).upsert_all (video.video_histories, False)
if video.video_tags is not None:
VideoTagDao (self.conn).upsert_all (video.video_tags, False)
if video.comments is not None:
CommentDao (self.conn).upsert_all (video.comments, False)
if video.video_histories is not None:
VideoHistoryDao (self.conn).upsert_all (video.video_histories)


def upsert_all ( def upsert_all (
self, self,
videos: list[VideoDto], videos: list[VideoDto],
with_relation_tables: bool = True,
with_relation_tables: bool,
) -> None:
for video in videos:
self.upsert (video, with_relation_tables)

def delete (
self,
video_ids: list[int],
at: datetime,
) -> None: ) -> None:
with self.conn.cursor () as c: with self.conn.cursor () as c:
for video in videos:
self.upsert (video, with_relation_tables)
c.execute ("""
UPDATE
videos
SET
deleted_at = %s
WHERE
id IN (%s)""", (at, (*video_ids,)))


def _create_dto_from_row ( def _create_dto_from_row (
self, self,
@@ -270,8 +313,8 @@ class VideoDao:
description = row['description'], description = row['description'],
uploaded_at = row['uploaded_at'], uploaded_at = row['uploaded_at'],
deleted_at = row['deleted_at']) deleted_at = row['deleted_at'])
if with_relation_tables:
video.video_tags = VideoTagDao (self.conn).fetch_by_video_id (cast (int, video.id_), False)
if with_relation_tables and video.id_ is not None:
video.video_tags = VideoTagDao (self.conn).fetch_by_video_id (video.id_, False)
for i in range (len (video.video_tags)): for i in range (len (video.video_tags)):
video.video_tags[i].video = video video.video_tags[i].video = video
video.comments = CommentDao (self.conn).fetch_by_video_id (video.id_, False) video.comments = CommentDao (self.conn).fetch_by_video_id (video.id_, False)
@@ -306,7 +349,7 @@ class VideoTagDao:
def fetch_by_video_id ( def fetch_by_video_id (
self, self,
video_id: int, video_id: int,
with_relation_tables: bool = True,
with_relation_tables: bool,
) -> list[VideoTagDto]: ) -> list[VideoTagDto]:
with self.conn.cursor () as c: with self.conn.cursor () as c:
c.execute (""" c.execute ("""
@@ -327,6 +370,140 @@ class VideoTagDao:
video_tags.append (self._create_dto_from_row (row, with_relation_tables)) video_tags.append (self._create_dto_from_row (row, with_relation_tables))
return video_tags return video_tags


def fetch_alive_by_video_id (
self,
video_id: int,
with_relation_tables: bool,
) -> list[VideoTagDto]:
with self.conn.cursor () as c:
c.execute ("""
SELECT
id,
video_id,
tag_id,
tagged_at,
untagged_at
FROM
video_tags
WHERE
video_id = %s
AND (untagged_at IS NULL)
ORDER BY
id""", video_id)
video_tags: list[VideoTagDto] = []
for row in c.fetchall ():
video_tags.append (self._create_dto_from_row (row, with_relation_tables))
return video_tags

def fetch_alive_by_ids (
self,
video_id: int,
tag_id: int,
with_relation_tables: bool,
) -> VideoTagDto | None:
with self.conn.cursor () as c:
c.execute ("""
SELECT
id,
video_id,
tag_id,
tagged_at,
untagged_at
FROM
video_tags
WHERE
video_id = %s
AND tag_id = %s""", (video_id, tag_id))
row = c.fetchone ()
if row is None:
return None
return self._create_dto_from_row (row, with_relation_tables)

def insert (
self,
video_tag: VideoTagDto,
with_relation_tables: bool,
) -> None:
with self.conn.cursor () as c:
c.execute ("""
INSERT INTO
video_tags(
video_id,
tag_id,
tagged_at,
untagged_at)
VALUES
(
%s,
%s,
%s,
%s)""", (video_tag.video_id, video_tag.tag_id,
video_tag.tagged_at, video_tag.untagged_at))
video_tag.id_ = c.lastrowid
if with_relation_tables:
if video_tag.video is not None:
VideoDao (self.conn).upsert (video_tag.video, True)
if video_tag.tag is not None:
TagDao (self.conn).upsert (video_tag.tag)

def upsert (
self,
video_tag: VideoTagDto,
with_relation_tables: bool,
) -> None:
with self.conn.cursor () as c:
c.execute ("""
INSERT INTO
video_tags(
video_id,
tag_id,
tagged_at,
untagged_at)
VALUES
(
%s,
%s,
%s,
%s)
ON DUPLICATE KEY UPDATE
video_id = VALUES(video_id),
tag_id = VALUES(tag_id),
tagged_at = VALUES(tagged_at),
untagged_at = VALUES(untagged_at)""", (video_tag.video_id,
video_tag.tag_id,
video_tag.tagged_at,
video_tag.untagged_at))
video_tag.id_ = c.lastrowid
if with_relation_tables:
if video_tag.video is not None:
VideoDao (self.conn).upsert (video_tag.video, True)
if video_tag.tag is not None:
TagDao (self.conn).upsert (video_tag.tag)

def upsert_all (
self,
video_tags: list[VideoTagDto],
with_relation_tables: bool,
) -> None:
for video_tag in video_tags:
self.upsert (video_tag, with_relation_tables)

def untag_all (
self,
video_id: int,
tag_ids: list[int],
now: datetime
) -> None:
with self.conn.cursor () as c:
c.execute ("""
UPDATE
video_tags
SET
untagged_at = %s
WHERE
video_id = %s
AND tag_ids IN (%s)""", (now, video_id, (*tag_ids,)))

def _create_dto_from_row ( def _create_dto_from_row (
self, self,
row, row,
@@ -338,28 +515,303 @@ class VideoTagDao:
tagged_at = row['tagged_at'], tagged_at = row['tagged_at'],
untagged_at = row['untagged_at']) untagged_at = row['untagged_at'])
if with_relation_tables: if with_relation_tables:
video_tag.video = VideoDao (self.conn).fetch (video_tag.video_id, True)
video_tag.tag = TagDao (self.conn).fetch (video_tag.tag_id, True)
video_tag.video = VideoDao (self.conn).find (video_tag.video_id, True)
video_tag.tag = TagDao (self.conn).find (video_tag.tag_id)
return video_tag return video_tag




@dataclass (slots = True) @dataclass (slots = True)
class VideoTagDto: class VideoTagDto:
video_id: int
tag_id: int
tagged_at: datetime tagged_at: datetime
id_: int | None = None id_: int | None = None
video_id: int | None = None
tag_id: int | None = None
untagged_at: datetime | DbNullType = DbNull untagged_at: datetime | DbNullType = DbNull
video: VideoDto | None = None video: VideoDto | None = None
tag: TagDto | None = None tag: TagDto | None = None




class TagDao:
def __init__ (
self,
conn,
):
self.conn = conn

def find (
self,
tag_id: int,
) -> TagDto | None:
with self.conn.cursor () as c:
c.execute ("""
SELECT
id,
name)
FROM
tags
WHERE
id = %s""", tag_id)
row = c.fetchone ()
if row is None:
return None
return self._create_dto_from_row (row)

def fetch_by_name (
self,
tag_name: str,
) -> TagDto | None:
with self.conn.cursor () as c:
c.execute ("""
SELECT
id,
name
FROM
tags
WHERE
name = %s""", tag_name)
row = c.fetchone ()
if row is None:
return None
return self._create_dto_from_row (row)

def insert (
self,
tag: TagDto,
) -> None:
with self.conn.cursor () as c:
c.execute ("""
INSERT INTO
tags(name)
VALUES
(%s)""", tag.name)
tag.id_ = c.lastrowid

def upsert (
self,
tag: TagDto,
) -> None:
with self.conn.cursor () as c:
c.execute ("""
INSERT INTO
tags(name)
VALUES
(%s)
ON DUPLICATE KEY UPDATE
name = VALUES(name)""", tag.name)
tag.id_ = c.lastrowid

def _create_dto_from_row (
self,
row,
) -> TagDto:
return TagDto (id_ = row['id'],
name = row['name'])


@dataclass (slots = True) @dataclass (slots = True)
class TagDto: class TagDto:
name: str name: str
id_: int | None = None id_: int | None = None




class VideoHistoryDao:
def __init__ (
self,
conn,
):
self.conn = conn

def fetch_by_video_id (
self,
video_id: int,
with_relation_tables: bool,
) -> list[VideoHistoryDto]:
with self.conn.cursor () as c:
c.execute ("""
SELECT
id,
video_id,
fetched_at,
views_count
FROM
video_histories
WHERE
video_id = %s""", video_id)
video_histories: list[VideoHistoryDto] = []
for row in c.fetchall ():
video_histories.append (self._create_dto_from_row (row, with_relation_tables))
return video_histories

def insert (
self,
video_history: VideoHistoryDto,
) -> None:
with self.conn.cursor () as c:
c.execute ("""
INSERT INTO
video_histories(
video_id,
fetched_at,
views_count)
VALUES
(
%s,
%s,
%s)""", (video_history.video_id,
video_history.fetched_at,
video_history.views_count))

def upsert (
self,
video_history: VideoHistoryDto,
) -> None:
with self.conn.cursor () as c:
c.execute ("""
INSERT INTO
video_histories(
video_id,
fetched_at,
views_count)
VALUES
(
%s,
%s,
%s)
ON DUPLICATE KEY UPDATE
video_id,
fetched_at,
views_count""", (video_history.video_id,
video_history.fetched_at,
video_history.views_count))

def upsert_all (
self,
video_histories: list[VideoHistoryDto],
) -> None:
for video_history in video_histories:
self.upsert (video_history)

def _create_dto_from_row (
self,
row,
with_relation_tables: bool,
) -> VideoHistoryDto:
video_history = VideoHistoryDto (id_ = row['id'],
video_id = row['video_id'],
fetched_at = row['fetched_at'],
views_count = row['views_count'])
if with_relation_tables:
video_history.video = VideoDao (self.conn).find (video_history.video_id, True)
return video_history


@dataclass (slots = True)
class VideoHistoryDto:
video_id: int
fetched_at: datetime
views_count: int
id_: int | None = None
video: VideoDto | None = None


class CommentDao:
def __init__ (
self,
conn,
):
self.conn = conn

def fetch_by_video_id (
self,
video_id: int,
with_relation_tables: bool,
) -> list[CommentDto]:
with self.conn.cursor () as c:
c.execute ("""
SELECT
id,
video_id,
comment_no,
user_id,
content,
posted_at,
nico_count,
vpos_ms
FROM
comments
WHERE
video_id = %s""", video_id)
comments: list[CommentDto] = []
for row in c.fetchall ():
comments.append (self._create_dto_from_row (row, with_relation_tables))
return comments

def upsert (
self,
comment: CommentDto,
with_relation_tables: bool,
) -> None:
with self.conn.cursor () as c:
c.execute ("""
INSERT INTO
comments(
video_id,
comment_no,
user_id,
content,
posted_at,
nico_count,
vpos_ms)
VALUES
(
%s,
%s,
%s,
%s,
%s,
%s,
%s)
ON DUPLICATE KEY UPDATE
video_id = VALUES(video_id),
comment_no = VALUES(comment_no),
user_id = VALUES(user_id),
content = VALUES(content),
posted_at = VALUES(posted_at),
nico_count = VALUES(nico_count),
vpos_ms = VALUES(vpos_ms)""", (comment.video_id,
comment.comment_no,
comment.user_id,
comment.content,
comment.posted_at,
comment.nico_count,
comment.vpos_ms))

def upsert_all (
self,
comments: list[CommentDto],
with_relation_tables: bool,
) -> None:
for comment in comments:
self.upsert (comment, with_relation_tables)

def _create_dto_from_row (
self,
row,
with_relation_tables: bool,
) -> CommentDto:
comment = CommentDto (id_ = row['id'],
video_id = row['video_id'],
comment_no = row['comment_no'],
user_id = row['user_id'],
content = row['content'],
posted_at = row['posted_at'],
nico_count = row['nico_count'],
vpos_ms = row['vpos_ms'])
if with_relation_tables:
comment.video = VideoDao (self.conn).find (comment.video_id, True)
return comment


@dataclass (slots = True) @dataclass (slots = True)
class CommentDto: class CommentDto:
video_id: int video_id: int
@@ -370,7 +822,7 @@ class CommentDto:
id_: int | None = None id_: int | None = None
nico_count: int = 0 nico_count: int = 0
vpos_ms: int | DbNullType = DbNull vpos_ms: int | DbNullType = DbNull
video: VideoDto | None = None




if __name__ == '__main__': if __name__ == '__main__':


Write Preview
Loading…
Cancel
Save