はじまりの大地

このコミットが含まれているのは:
2024-07-15 09:14:04 +09:00
コミット 6632905f32
3501個のファイルの変更1439465行の追加0行の削除
+3
ファイルの表示
@@ -0,0 +1,3 @@
export * from './video-viewer-counters.js'
export * from './video-viewer-stats.js'
export * from './video-views.js'
+248
ファイルの表示
@@ -0,0 +1,248 @@
import { isTestOrDevInstance, isUsingViewersFederationV2 } from '@peertube/peertube-node-utils'
import { exists } from '@server/helpers/custom-validators/misc.js'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { VIEW_LIFETIME } from '@server/initializers/constants.js'
import { sendView } from '@server/lib/activitypub/send/send-view.js'
import { PeerTubeSocket } from '@server/lib/peertube-socket.js'
import { getServerActor } from '@server/models/application/application.js'
import { VideoModel } from '@server/models/video/video.js'
import { MVideo, MVideoImmutable } from '@server/types/models/index.js'
const lTags = loggerTagsFactory('views')
export type ViewerScope = 'local' | 'remote'
export type VideoScope = 'local' | 'remote'
type Viewer = {
expires: number
id: string
viewerScope: ViewerScope
videoScope: VideoScope
viewerCount: number
lastFederation?: number
}
export class VideoViewerCounters {
// expires is new Date().getTime()
private readonly viewersPerVideo = new Map<number, Viewer[]>()
private readonly idToViewer = new Map<string, Viewer>()
private processingViewerCounters = false
constructor () {
setInterval(() => this.updateVideoViewersCount(), VIEW_LIFETIME.VIEWER_COUNTER)
}
// ---------------------------------------------------------------------------
async addLocalViewer (options: {
video: MVideoImmutable
sessionId: string
}) {
const { video, sessionId } = options
logger.debug('Adding local viewer to video viewers counter %s.', video.uuid, { ...lTags(video.uuid) })
const viewerId = sessionId + '-' + video.uuid
const viewer = this.idToViewer.get(viewerId)
if (viewer) {
viewer.expires = this.buildViewerExpireTime()
await this.federateViewerIfNeeded(video, viewer)
return false
}
const newViewer = this.addViewerToVideo({ viewerId, video, viewerScope: 'local', viewerCount: 1 })
await this.federateViewerIfNeeded(video, newViewer)
return true
}
addRemoteViewerOnLocalVideo (options: {
video: MVideo
viewerId: string
viewerExpires: Date
}) {
const { video, viewerExpires, viewerId } = options
logger.debug('Adding remote viewer to local video %s.', video.uuid, { viewerId, viewerExpires, ...lTags(video.uuid) })
this.addViewerToVideo({ video, viewerExpires, viewerId, viewerScope: 'remote', viewerCount: 1 })
return true
}
addRemoteViewerOnRemoteVideo (options: {
video: MVideo
viewerId: string
viewerExpires: Date
viewerResultCounter?: number
}) {
const { video, viewerExpires, viewerId, viewerResultCounter } = options
logger.debug(
'Adding remote viewer to remote video %s.', video.uuid,
{ viewerId, viewerResultCounter, viewerExpires, ...lTags(video.uuid) }
)
this.addViewerToVideo({
video,
viewerExpires,
viewerId,
viewerScope: 'remote',
// The origin server sends a summary of all viewers, so we can replace our local copy
replaceCurrentViewers: exists(viewerResultCounter),
viewerCount: viewerResultCounter ?? 1
})
return true
}
// ---------------------------------------------------------------------------
getTotalViewers (options: {
viewerScope: ViewerScope
videoScope: VideoScope
}) {
let total = 0
for (const viewers of this.viewersPerVideo.values()) {
total += viewers.filter(v => v.viewerScope === options.viewerScope && v.videoScope === options.videoScope)
.reduce((p, c) => p + c.viewerCount, 0)
}
return total
}
getTotalViewersOf (video: MVideoImmutable) {
const viewers = this.viewersPerVideo.get(video.id)
return viewers?.reduce((p, c) => p + c.viewerCount, 0) || 0
}
buildViewerExpireTime () {
return new Date().getTime() + VIEW_LIFETIME.VIEWER_COUNTER
}
// ---------------------------------------------------------------------------
private addViewerToVideo (options: {
video: MVideoImmutable
viewerId: string
viewerScope: ViewerScope
viewerCount: number
replaceCurrentViewers?: boolean
viewerExpires?: Date
}) {
const { video, viewerExpires, viewerId, viewerScope, viewerCount, replaceCurrentViewers } = options
let watchers = this.viewersPerVideo.get(video.id)
if (!watchers || replaceCurrentViewers) {
watchers = []
this.viewersPerVideo.set(video.id, watchers)
}
const expires = viewerExpires
? viewerExpires.getTime()
: this.buildViewerExpireTime()
const videoScope: VideoScope = video.remote
? 'remote'
: 'local'
const viewer = { id: viewerId, expires, videoScope, viewerScope, viewerCount }
watchers.push(viewer)
this.idToViewer.set(viewerId, viewer)
this.notifyClients(video)
return viewer
}
private async updateVideoViewersCount () {
if (this.processingViewerCounters) return
this.processingViewerCounters = true
if (!isTestOrDevInstance()) {
logger.debug('Updating video viewer counters.', lTags())
}
try {
for (const videoId of this.viewersPerVideo.keys()) {
const notBefore = new Date().getTime()
const viewers = this.viewersPerVideo.get(videoId)
// Only keep not expired viewers
const newViewers: Viewer[] = []
// Filter new viewers
for (const viewer of viewers) {
if (viewer.expires > notBefore) {
newViewers.push(viewer)
} else {
this.idToViewer.delete(viewer.id)
}
}
if (newViewers.length === 0) this.viewersPerVideo.delete(videoId)
else this.viewersPerVideo.set(videoId, newViewers)
const video = await VideoModel.loadImmutableAttributes(videoId)
if (video) {
this.notifyClients(video)
// Let total viewers expire on remote instances if there are no more viewers
if (video.remote === false && newViewers.length !== 0) {
await this.federateTotalViewers(video)
}
}
}
} catch (err) {
logger.error('Error in video clean viewers scheduler.', { err, ...lTags() })
}
this.processingViewerCounters = false
}
private notifyClients (video: MVideoImmutable) {
const totalViewers = this.getTotalViewersOf(video)
PeerTubeSocket.Instance.sendVideoViewsUpdate(video, totalViewers)
logger.debug('Video viewers update for %s is %d.', video.url, totalViewers, lTags())
}
private async federateViewerIfNeeded (video: MVideoImmutable, viewer: Viewer) {
// Federate the viewer if it's been a "long" time we did not
const now = new Date().getTime()
const federationLimit = now - (VIEW_LIFETIME.VIEWER_COUNTER * 0.75)
if (viewer.lastFederation && viewer.lastFederation > federationLimit) return
if (video.remote === false && isUsingViewersFederationV2()) return
await sendView({
byActor: await getServerActor(),
video,
viewersCount: 1,
viewerIdentifier: viewer.id
})
viewer.lastFederation = now
}
private async federateTotalViewers (video: MVideoImmutable) {
if (!isUsingViewersFederationV2()) return
await sendView({
byActor: await getServerActor(),
video,
viewersCount: this.getTotalViewersOf(video),
viewerIdentifier: video.uuid
})
}
}
+260
ファイルの表示
@@ -0,0 +1,260 @@
import { VideoViewEvent } from '@peertube/peertube-models'
import { isTestOrDevInstance } from '@peertube/peertube-node-utils'
import { GeoIP } from '@server/helpers/geo-ip.js'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { MAX_LOCAL_VIEWER_WATCH_SECTIONS, VIEWER_SYNC_REDIS, VIEW_LIFETIME } from '@server/initializers/constants.js'
import { sequelizeTypescript } from '@server/initializers/database.js'
import { sendCreateWatchAction } from '@server/lib/activitypub/send/index.js'
import { getLocalVideoViewerActivityPubUrl } from '@server/lib/activitypub/url.js'
import { Redis } from '@server/lib/redis.js'
import { VideoModel } from '@server/models/video/video.js'
import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section.js'
import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer.js'
import { MVideo, MVideoImmutable } from '@server/types/models/index.js'
import { Transaction } from 'sequelize'
const lTags = loggerTagsFactory('views')
type LocalViewerStats = {
firstUpdated: number // Date.getTime()
lastUpdated: number // Date.getTime()
watchSections: {
start: number
end: number
}[]
watchTime: number
country: string
subdivisionName: string
videoId: number
}
export class VideoViewerStats {
private processingViewersStats = false
private processingRedisWrites = false
private readonly viewerCache = new Map<string, LocalViewerStats>()
private readonly redisPendingWrites = new Map<string, { sessionId: string, videoId: number, stats: LocalViewerStats }>()
constructor () {
setInterval(() => this.processViewerStats(), VIEW_LIFETIME.VIEWER_STATS)
setInterval(() => this.syncRedisWrites(), VIEWER_SYNC_REDIS)
}
// ---------------------------------------------------------------------------
async addLocalViewer (options: {
video: MVideoImmutable
currentTime: number
ip: string
sessionId: string
viewEvent?: VideoViewEvent
}) {
const { video, ip, viewEvent, currentTime, sessionId } = options
logger.debug(
'Adding local viewer to video stats %s.', video.uuid,
{ currentTime, viewEvent, sessionId, ...lTags(video.uuid) }
)
const nowMs = new Date().getTime()
let stats: LocalViewerStats = await this.getLocalVideoViewer({ sessionId, videoId: video.id })
if (stats && stats.watchSections.length >= MAX_LOCAL_VIEWER_WATCH_SECTIONS) {
logger.warn(
'Too much watch section to store for a viewer, skipping this one',
{ sessionId, currentTime, viewEvent, ...lTags(video.uuid) }
)
return
}
if (!stats) {
const { country, subdivisionName } = await GeoIP.Instance.safeIPISOLookup(ip)
stats = {
firstUpdated: nowMs,
lastUpdated: nowMs,
watchSections: [],
watchTime: 0,
country,
subdivisionName,
videoId: video.id
}
}
stats.lastUpdated = nowMs
if (viewEvent === 'seek' || stats.watchSections.length === 0) {
stats.watchSections.push({
start: currentTime,
end: currentTime
})
} else {
const lastSection = stats.watchSections[stats.watchSections.length - 1]
if (lastSection.start > currentTime) {
logger.debug('Invalid end watch section %d. Last start record was at %d. Starting a new section.', currentTime, lastSection.start)
stats.watchSections.push({
start: currentTime,
end: currentTime
})
} else {
lastSection.end = currentTime
}
}
stats.watchTime = this.buildWatchTimeFromSections(stats.watchSections)
logger.debug('Set local video viewer stats for video %s.', video.uuid, { stats, ...lTags(video.uuid) })
this.setLocalVideoViewer(sessionId, video.id, stats)
}
// ---------------------------------------------------------------------------
async getWatchTime (videoId: number, sessionId: string) {
const stats: LocalViewerStats = await this.getLocalVideoViewer({ sessionId, videoId })
return stats?.watchTime || 0
}
// ---------------------------------------------------------------------------
async processViewerStats () {
if (this.processingViewersStats) return
this.processingViewersStats = true
if (!isTestOrDevInstance()) logger.info('Processing viewer statistics.', lTags())
const now = new Date().getTime()
try {
await this.syncRedisWrites()
const allKeys = await Redis.Instance.listLocalVideoViewerKeys()
for (const key of allKeys) {
const stats: LocalViewerStats = await this.getLocalVideoViewerByKey(key)
// Process expired stats
if (stats.lastUpdated > now - VIEW_LIFETIME.VIEWER_STATS) {
continue
}
try {
await sequelizeTypescript.transaction(async t => {
const video = await VideoModel.load(stats.videoId, t)
if (!video) return
const statsModel = await this.saveViewerStats(video, stats, t)
if (statsModel && video.remote) {
await sendCreateWatchAction(statsModel, t)
}
})
await this.deleteLocalVideoViewersKeys(key)
} catch (err) {
logger.error('Cannot process viewer stats for Redis key %s.', key, { err, stats, ...lTags() })
}
}
} catch (err) {
logger.error('Error in video save viewers stats scheduler.', { err, ...lTags() })
}
this.processingViewersStats = false
}
private async saveViewerStats (video: MVideo, stats: LocalViewerStats, transaction: Transaction) {
if (stats.watchTime === 0) return
const statsModel = new LocalVideoViewerModel({
startDate: new Date(stats.firstUpdated),
endDate: new Date(stats.lastUpdated),
watchTime: stats.watchTime,
country: stats.country,
subdivisionName: stats.subdivisionName,
videoId: video.id
})
statsModel.url = getLocalVideoViewerActivityPubUrl(statsModel)
statsModel.Video = video as VideoModel
await statsModel.save({ transaction })
statsModel.WatchSections = await LocalVideoViewerWatchSectionModel.bulkCreateSections({
localVideoViewerId: statsModel.id,
watchSections: stats.watchSections,
transaction
})
return statsModel
}
private buildWatchTimeFromSections (sections: { start: number, end: number }[]) {
return sections.reduce((p, current) => p + (current.end - current.start), 0)
}
/**
*
* Redis calls can be expensive so try to cache things in front of it
*
*/
private getLocalVideoViewer (options: {
sessionId: string
videoId: number
}): Promise<LocalViewerStats> {
const { viewerKey } = Redis.Instance.generateLocalVideoViewerKeys(options.sessionId, options.videoId)
return this.getLocalVideoViewerByKey(viewerKey)
}
private getLocalVideoViewerByKey (key: string): Promise<LocalViewerStats> {
const viewer = this.viewerCache.get(key)
if (viewer) return Promise.resolve(viewer)
return Redis.Instance.getLocalVideoViewer({ key })
}
private setLocalVideoViewer (sessionId: string, videoId: number, stats: LocalViewerStats) {
const { viewerKey } = Redis.Instance.generateLocalVideoViewerKeys(sessionId, videoId)
this.viewerCache.set(viewerKey, stats)
this.redisPendingWrites.set(viewerKey, { sessionId, videoId, stats })
}
private deleteLocalVideoViewersKeys (key: string) {
this.viewerCache.delete(key)
return Redis.Instance.deleteLocalVideoViewersKeys(key)
}
private async syncRedisWrites () {
if (this.processingRedisWrites) return
this.processingRedisWrites = true
for (const [ key, pendingWrite ] of this.redisPendingWrites) {
const { sessionId, videoId, stats } = pendingWrite
this.redisPendingWrites.delete(key)
try {
await Redis.Instance.setLocalVideoViewer(sessionId, videoId, stats)
} catch (err) {
logger.error('Cannot write viewer into redis', { sessionId, videoId, stats, err })
}
}
this.processingRedisWrites = false
}
}
+94
ファイルの表示
@@ -0,0 +1,94 @@
import { buildUUID } from '@peertube/peertube-node-utils'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { VIEW_LIFETIME } from '@server/initializers/constants.js'
import { sendView } from '@server/lib/activitypub/send/send-view.js'
import { getCachedVideoDuration } from '@server/lib/video.js'
import { getServerActor } from '@server/models/application/application.js'
import { MVideo, MVideoImmutable } from '@server/types/models/index.js'
import { LRUCache } from 'lru-cache'
import { Redis } from '../../redis.js'
import { CONFIG } from '@server/initializers/config.js'
const lTags = loggerTagsFactory('views')
export class VideoViews {
private readonly viewsCache = new LRUCache<string, boolean>({
max: 10_000,
ttl: VIEW_LIFETIME.VIEW
})
async addLocalView (options: {
video: MVideoImmutable
sessionId: string
watchTime: number
}) {
const { video, sessionId, watchTime } = options
logger.debug('Adding local view to video %s.', video.uuid, { watchTime, ...lTags(video.uuid) })
if (!await this.hasEnoughWatchTime(video, watchTime)) return false
const viewExists = await this.doesVideoSessionIdViewExist(sessionId, video.uuid)
if (viewExists) return false
await this.setSessionIdVideoView(sessionId, video.uuid)
await this.addView(video)
await sendView({ byActor: await getServerActor(), video, viewerIdentifier: buildUUID() })
return true
}
async addRemoteView (options: {
video: MVideo
}) {
const { video } = options
logger.debug('Adding remote view to video %s.', video.uuid, { ...lTags(video.uuid) })
await this.addView(video)
return true
}
// ---------------------------------------------------------------------------
private async addView (video: MVideoImmutable) {
const promises: Promise<any>[] = []
if (video.isOwned()) {
promises.push(Redis.Instance.addLocalVideoView(video.id))
}
promises.push(Redis.Instance.addVideoViewStats(video.id))
await Promise.all(promises)
}
private async hasEnoughWatchTime (video: MVideoImmutable, watchTime: number) {
const { duration, isLive } = await getCachedVideoDuration(video.id)
const countViewAfterSeconds = CONFIG.VIEWS.VIDEOS.COUNT_VIEW_AFTER / 1000 // Config is in ms
if (isLive || duration >= countViewAfterSeconds) return watchTime >= countViewAfterSeconds
// Check more than 50% of the video is watched
return duration / watchTime < 2
}
private doesVideoSessionIdViewExist (sessionId: string, videoUUID: string) {
const key = Redis.Instance.generateSessionIdViewKey(sessionId, videoUUID)
const value = this.viewsCache.has(key)
if (value === true) return Promise.resolve(true)
return Redis.Instance.doesVideoSessionIdViewExist(sessionId, videoUUID)
}
private setSessionIdVideoView (sessionId: string, videoUUID: string) {
const key = Redis.Instance.generateSessionIdViewKey(sessionId, videoUUID)
this.viewsCache.set(key, true)
return Redis.Instance.setSessionIdVideoView(sessionId, videoUUID)
}
}