|
@@ -1,3 +1,6 @@ |
|
|
|
|
|
# pylint: disable = missing-class-docstring |
|
|
|
|
|
# pylint: disable = missing-function-docstring |
|
|
|
|
|
|
|
|
""" |
|
|
""" |
|
|
日次で実行し,ぼざクリ DB を最新に更新する. |
|
|
日次で実行し,ぼざクリ DB を最新に更新する. |
|
|
""" |
|
|
""" |
|
@@ -10,13 +13,11 @@ import random |
|
|
import string |
|
|
import string |
|
|
import time |
|
|
import time |
|
|
import unicodedata |
|
|
import unicodedata |
|
|
from dataclasses import dataclass |
|
|
|
|
|
from datetime import date, datetime, timedelta |
|
|
from datetime import date, datetime, timedelta |
|
|
from typing import Any, Type, TypedDict, cast |
|
|
|
|
|
|
|
|
from typing import Any, TypedDict, cast |
|
|
|
|
|
|
|
|
import requests |
|
|
import requests |
|
|
from eloquent import DatabaseManager, Model |
|
|
from eloquent import DatabaseManager, Model |
|
|
from eloquent.orm.relations.dynamic_property import DynamicProperty |
|
|
|
|
|
|
|
|
|
|
|
config: dict[str, DbConfig] = { 'mysql': { 'driver': 'mysql', |
|
|
config: dict[str, DbConfig] = { 'mysql': { 'driver': 'mysql', |
|
|
'host': 'localhost', |
|
|
'host': 'localhost', |
|
@@ -58,6 +59,8 @@ def update_tables ( |
|
|
video_history.views_count = datum['viewCounter'] |
|
|
video_history.views_count = datum['viewCounter'] |
|
|
video_history.save () |
|
|
video_history.save () |
|
|
video_tags = video.video_tags.where_not_null ('untagged_at').get () |
|
|
video_tags = video.video_tags.where_not_null ('untagged_at').get () |
|
|
|
|
|
tag: Tag | None |
|
|
|
|
|
video_tag: VideoTag | None |
|
|
for video_tag in video_tags: |
|
|
for video_tag in video_tags: |
|
|
tag = video_tag.tag |
|
|
tag = video_tag.tag |
|
|
if (tag is not None |
|
|
if (tag is not None |
|
@@ -70,10 +73,10 @@ def update_tables ( |
|
|
tag = Tag () |
|
|
tag = Tag () |
|
|
tag.name = tag_name |
|
|
tag.name = tag_name |
|
|
tag.save () |
|
|
tag.save () |
|
|
video_tag = (Video.where ('video_id', video.id) |
|
|
|
|
|
.where ('tag_id', tag.id) |
|
|
|
|
|
.where_null ('untagged_at') |
|
|
|
|
|
.first ()) |
|
|
|
|
|
|
|
|
video_tag = (VideoTag.where ('video_id', video.id) |
|
|
|
|
|
.where ('tag_id', tag.id) |
|
|
|
|
|
.where_null ('untagged_at') |
|
|
|
|
|
.first ()) |
|
|
if video_tag is None: |
|
|
if video_tag is None: |
|
|
video_tag = VideoTag () |
|
|
video_tag = VideoTag () |
|
|
video_tag.video_id = video.id |
|
|
video_tag.video_id = video.id |
|
@@ -173,12 +176,15 @@ def search_nico_by_tags ( |
|
|
while to <= today: |
|
|
while to <= today: |
|
|
time.sleep (1.2) |
|
|
time.sleep (1.2) |
|
|
until = to + timedelta (days = 14) |
|
|
until = to + timedelta (days = 14) |
|
|
|
|
|
# pylint: disable = consider-using-f-string |
|
|
query_filter = json.dumps ({ 'type': 'or', |
|
|
query_filter = json.dumps ({ 'type': 'or', |
|
|
'filters': [ |
|
|
'filters': [ |
|
|
{ 'type': 'range', |
|
|
{ 'type': 'range', |
|
|
'field': 'startTime', |
|
|
'field': 'startTime', |
|
|
'from': '%04d-%02d-%02dT00:00:00+09:00' % (to.year, to.month, to.day), |
|
|
|
|
|
'to': '%04d-%02d-%02dT23:59:59+09:00' % (until.year, until.month, until.day), |
|
|
|
|
|
|
|
|
'from': ('%04d-%02d-%02dT00:00:00+09:00' |
|
|
|
|
|
% (to.year, to.month, to.day)), |
|
|
|
|
|
'to': ('%04d-%02d-%02dT23:59:59+09:00' |
|
|
|
|
|
% (until.year, until.month, until.day)), |
|
|
'include_lower': True }] }) |
|
|
'include_lower': True }] }) |
|
|
params: VideoSearchParam = { 'q': ' OR '.join (tags), |
|
|
params: VideoSearchParam = { 'q': ' OR '.join (tags), |
|
|
'targets': 'tagsExact', |
|
|
'targets': 'tagsExact', |
|
@@ -202,8 +208,9 @@ def search_nico_by_tags ( |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Comment (Model): |
|
|
class Comment (Model): |
|
|
__timestamps__ = False |
|
|
|
|
|
|
|
|
# pylint: disable = too-many-instance-attributes |
|
|
|
|
|
|
|
|
|
|
|
id: int |
|
|
video_id: int |
|
|
video_id: int |
|
|
comment_no: int |
|
|
comment_no: int |
|
|
user_id: int |
|
|
user_id: int |
|
@@ -212,16 +219,18 @@ class Comment (Model): |
|
|
nico_count: int |
|
|
nico_count: int |
|
|
vpos_ms: int |
|
|
vpos_ms: int |
|
|
|
|
|
|
|
|
|
|
|
__timestamps__ = False |
|
|
|
|
|
|
|
|
@property |
|
|
@property |
|
|
def video ( |
|
|
def video ( |
|
|
self, |
|
|
self, |
|
|
) -> DynamicProperty: |
|
|
|
|
|
|
|
|
) -> Video: |
|
|
return self.belongs_to (Video) |
|
|
return self.belongs_to (Video) |
|
|
|
|
|
|
|
|
@property |
|
|
@property |
|
|
def user ( |
|
|
def user ( |
|
|
self, |
|
|
self, |
|
|
) -> DynamicProperty: |
|
|
|
|
|
|
|
|
) -> User: |
|
|
return self.belongs_to (User) |
|
|
return self.belongs_to (User) |
|
|
|
|
|
|
|
|
def upsert ( |
|
|
def upsert ( |
|
@@ -232,58 +241,62 @@ class Comment (Model): |
|
|
.first ()) |
|
|
.first ()) |
|
|
if row is not None: |
|
|
if row is not None: |
|
|
self.id = row.id |
|
|
self.id = row.id |
|
|
|
|
|
self.__exists = True # pylint: disable = unused-private-member |
|
|
self.save () |
|
|
self.save () |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Tag (Model): |
|
|
class Tag (Model): |
|
|
__timestamps__ = False |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
id: int |
|
|
name: str |
|
|
name: str |
|
|
|
|
|
|
|
|
|
|
|
__timestamps__ = False |
|
|
|
|
|
|
|
|
@property |
|
|
@property |
|
|
def video_tags ( |
|
|
def video_tags ( |
|
|
self, |
|
|
self, |
|
|
) -> DynamicProperty: |
|
|
|
|
|
|
|
|
) -> VideoTag: |
|
|
return self.has_many (VideoTag) |
|
|
return self.has_many (VideoTag) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class User (Model): |
|
|
class User (Model): |
|
|
__timestamps__ = False |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
id: int |
|
|
code: str |
|
|
code: str |
|
|
|
|
|
|
|
|
|
|
|
__timestamps__ = False |
|
|
|
|
|
|
|
|
@property |
|
|
@property |
|
|
def comments ( |
|
|
def comments ( |
|
|
self, |
|
|
self, |
|
|
) -> DynamicProperty: |
|
|
|
|
|
|
|
|
) -> Comment: |
|
|
return self.has_many (Comment) |
|
|
return self.has_many (Comment) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Video (Model): |
|
|
class Video (Model): |
|
|
__timestamps__ = False |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
id: int |
|
|
code: str |
|
|
code: str |
|
|
title: str |
|
|
title: str |
|
|
description: str |
|
|
description: str |
|
|
uploaded_at: datetime |
|
|
uploaded_at: datetime |
|
|
deleted_at: datetime | None |
|
|
deleted_at: datetime | None |
|
|
|
|
|
|
|
|
|
|
|
__timestamps__ = False |
|
|
|
|
|
|
|
|
@property |
|
|
@property |
|
|
def video_histories ( |
|
|
def video_histories ( |
|
|
self, |
|
|
self, |
|
|
) -> DynamicProperty: |
|
|
|
|
|
|
|
|
) -> VideoHistory: |
|
|
return self.has_many (VideoHistory) |
|
|
return self.has_many (VideoHistory) |
|
|
|
|
|
|
|
|
@property |
|
|
@property |
|
|
def video_tags ( |
|
|
def video_tags ( |
|
|
self, |
|
|
self, |
|
|
) -> DynamicProperty: |
|
|
|
|
|
|
|
|
) -> VideoTag: |
|
|
return self.has_many (VideoTag) |
|
|
return self.has_many (VideoTag) |
|
|
|
|
|
|
|
|
@property |
|
|
@property |
|
|
def comments ( |
|
|
def comments ( |
|
|
self, |
|
|
self, |
|
|
) -> DynamicProperty: |
|
|
|
|
|
|
|
|
) -> Comment: |
|
|
return self.has_many (Comment) |
|
|
return self.has_many (Comment) |
|
|
|
|
|
|
|
|
def upsert ( |
|
|
def upsert ( |
|
@@ -292,20 +305,22 @@ class Video (Model): |
|
|
row = Video.where ('code', self.code).first () |
|
|
row = Video.where ('code', self.code).first () |
|
|
if row is not None: |
|
|
if row is not None: |
|
|
self.id = row.id |
|
|
self.id = row.id |
|
|
|
|
|
self.__exists = True # pylint: disable = unused-private-member |
|
|
self.save () |
|
|
self.save () |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class VideoHistory (Model): |
|
|
class VideoHistory (Model): |
|
|
__timestamps__ = False |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
id: int |
|
|
video_id: int |
|
|
video_id: int |
|
|
fetched_at: date |
|
|
fetched_at: date |
|
|
views_count: int |
|
|
views_count: int |
|
|
|
|
|
|
|
|
|
|
|
__timestamps__ = False |
|
|
|
|
|
|
|
|
@property |
|
|
@property |
|
|
def video ( |
|
|
def video ( |
|
|
self, |
|
|
self, |
|
|
) -> DynamicProperty: |
|
|
|
|
|
|
|
|
) -> Video: |
|
|
return self.belongs_to (Video) |
|
|
return self.belongs_to (Video) |
|
|
|
|
|
|
|
|
def upsert ( |
|
|
def upsert ( |
|
@@ -316,27 +331,29 @@ class VideoHistory (Model): |
|
|
.first ()) |
|
|
.first ()) |
|
|
if row is not None: |
|
|
if row is not None: |
|
|
self.id = row.id |
|
|
self.id = row.id |
|
|
|
|
|
self.__exists = True # pylint: disable = unused-private-member |
|
|
self.save () |
|
|
self.save () |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class VideoTag (Model): |
|
|
class VideoTag (Model): |
|
|
__timestamps__ = False |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
id: int |
|
|
video_id: int |
|
|
video_id: int |
|
|
tag_id: int |
|
|
tag_id: int |
|
|
tagged_at: date |
|
|
tagged_at: date |
|
|
untagged_at: date | None |
|
|
untagged_at: date | None |
|
|
|
|
|
|
|
|
|
|
|
__timestamps__ = False |
|
|
|
|
|
|
|
|
@property |
|
|
@property |
|
|
def video ( |
|
|
def video ( |
|
|
self, |
|
|
self, |
|
|
) -> DynamicProperty: |
|
|
|
|
|
|
|
|
) -> Video: |
|
|
return self.belongs_to (Video) |
|
|
return self.belongs_to (Video) |
|
|
|
|
|
|
|
|
@property |
|
|
@property |
|
|
def tag ( |
|
|
def tag ( |
|
|
self, |
|
|
self, |
|
|
) -> DynamicProperty: |
|
|
|
|
|
|
|
|
) -> Tag: |
|
|
return self.belongs_to (Tag) |
|
|
return self.belongs_to (Tag) |
|
|
|
|
|
|
|
|
def upsert ( |
|
|
def upsert ( |
|
@@ -347,6 +364,7 @@ class VideoTag (Model): |
|
|
.first ()) |
|
|
.first ()) |
|
|
if row is not None: |
|
|
if row is not None: |
|
|
self.id = row.id |
|
|
self.id = row.id |
|
|
|
|
|
self.__exists = True # pylint: disable = unused-private-member |
|
|
self.save () |
|
|
self.save () |
|
|
|
|
|
|
|
|
|
|
|
|
|
|