はじまりの大地

このコミットが含まれているのは:
2024-07-15 09:14:04 +09:00
コミット 6632905f32
3501個のファイルの変更1439465行の追加0行の削除
@@ -0,0 +1,51 @@
import { Meter } from '@opentelemetry/api'
export class BittorrentTrackerObserversBuilder {
constructor (private readonly meter: Meter, private readonly trackerServer: any) {
}
buildObservers () {
const activeInfohashes = this.meter.createObservableGauge('peertube_bittorrent_tracker_active_infohashes_total', {
description: 'Total active infohashes in the PeerTube BitTorrent Tracker'
})
const inactiveInfohashes = this.meter.createObservableGauge('peertube_bittorrent_tracker_inactive_infohashes_total', {
description: 'Total inactive infohashes in the PeerTube BitTorrent Tracker'
})
const peers = this.meter.createObservableGauge('peertube_bittorrent_tracker_peers_total', {
description: 'Total peers in the PeerTube BitTorrent Tracker'
})
this.meter.addBatchObservableCallback(observableResult => {
const infohashes = Object.keys(this.trackerServer.torrents)
const counters = {
activeInfohashes: 0,
inactiveInfohashes: 0,
peers: 0,
uncompletedPeers: 0
}
for (const infohash of infohashes) {
const content = this.trackerServer.torrents[infohash]
const peers = content.peers
if (peers.keys.length !== 0) counters.activeInfohashes++
else counters.inactiveInfohashes++
for (const peerId of peers.keys) {
const peer = peers.peek(peerId)
if (peer == null) return
counters.peers++
}
}
observableResult.observe(activeInfohashes, counters.activeInfohashes)
observableResult.observe(inactiveInfohashes, counters.inactiveInfohashes)
observableResult.observe(peers, counters.peers)
}, [ activeInfohashes, inactiveInfohashes, peers ])
}
}
+7
ファイルの表示
@@ -0,0 +1,7 @@
export * from './bittorrent-tracker-observers-builder.js'
export * from './lives-observers-builder.js'
export * from './job-queue-observers-builder.js'
export * from './nodejs-observers-builder.js'
export * from './playback-metrics.js'
export * from './stats-observers-builder.js'
export * from './viewers-observers-builder.js'
+24
ファイルの表示
@@ -0,0 +1,24 @@
import { Meter } from '@opentelemetry/api'
import { JobQueue } from '@server/lib/job-queue/index.js'
export class JobQueueObserversBuilder {
constructor (private readonly meter: Meter) {
}
buildObservers () {
this.meter.createObservableGauge('peertube_job_queue_total', {
description: 'Total jobs in the PeerTube job queue'
}).addCallback(async observableResult => {
const stats = await JobQueue.Instance.getStats()
for (const { jobType, counts } of stats) {
for (const state of Object.keys(counts)) {
observableResult.observe(counts[state], { jobType, state })
}
}
})
}
}
+21
ファイルの表示
@@ -0,0 +1,21 @@
import { Meter } from '@opentelemetry/api'
import { VideoModel } from '@server/models/video/video.js'
export class LivesObserversBuilder {
constructor (private readonly meter: Meter) {
}
buildObservers () {
this.meter.createObservableGauge('peertube_running_lives_total', {
description: 'Total running lives on the instance'
}).addCallback(async observableResult => {
const local = await VideoModel.countLives({ remote: false, mode: 'published' })
const remote = await VideoModel.countLives({ remote: true, mode: 'published' })
observableResult.observe(local, { liveOrigin: 'local' })
observableResult.observe(remote, { liveOrigin: 'remote' })
})
}
}
+197
ファイルの表示
@@ -0,0 +1,197 @@
import { readdir } from 'fs/promises'
import { constants, NodeGCPerformanceDetail, PerformanceObserver } from 'perf_hooks'
import * as process from 'process'
import { Meter, ObservableResult } from '@opentelemetry/api'
import { ExplicitBucketHistogramAggregation } from '@opentelemetry/sdk-metrics'
import { View } from '@opentelemetry/sdk-metrics/build/src/view/View.js'
import { logger } from '@server/helpers/logger.js'
// Thanks to https://github.com/siimon/prom-client
// We took their logic and adapter it for opentelemetry
// Try to keep consistency with their metric name/description so it's easier to process (grafana dashboard template etc)
export class NodeJSObserversBuilder {
constructor (private readonly meter: Meter) {
}
static getViews () {
return [
new View({
aggregation: new ExplicitBucketHistogramAggregation([ 0.001, 0.01, 0.1, 1, 2, 5 ]),
instrumentName: 'nodejs_gc_duration_seconds'
})
]
}
buildObservers () {
this.buildCPUObserver()
this.buildMemoryObserver()
this.buildHandlesObserver()
this.buildFileDescriptorsObserver()
this.buildGCObserver()
this.buildEventLoopLagObserver()
this.buildLibUVActiveRequestsObserver()
this.buildActiveResourcesObserver()
}
private buildCPUObserver () {
const cpuTotal = this.meter.createObservableCounter('process_cpu_seconds_total', {
description: 'Total user and system CPU time spent in seconds.'
})
const cpuUser = this.meter.createObservableCounter('process_cpu_user_seconds_total', {
description: 'Total user CPU time spent in seconds.'
})
const cpuSystem = this.meter.createObservableCounter('process_cpu_system_seconds_total', {
description: 'Total system CPU time spent in seconds.'
})
let lastCpuUsage = process.cpuUsage()
this.meter.addBatchObservableCallback(observableResult => {
const cpuUsage = process.cpuUsage()
const userUsageMicros = cpuUsage.user - lastCpuUsage.user
const systemUsageMicros = cpuUsage.system - lastCpuUsage.system
lastCpuUsage = cpuUsage
observableResult.observe(cpuTotal, (userUsageMicros + systemUsageMicros) / 1e6)
observableResult.observe(cpuUser, userUsageMicros / 1e6)
observableResult.observe(cpuSystem, systemUsageMicros / 1e6)
}, [ cpuTotal, cpuUser, cpuSystem ])
}
private buildMemoryObserver () {
this.meter.createObservableGauge('nodejs_memory_usage_bytes', {
description: 'Memory'
}).addCallback(observableResult => {
const current = process.memoryUsage()
observableResult.observe(current.heapTotal, { memoryType: 'heapTotal' })
observableResult.observe(current.heapUsed, { memoryType: 'heapUsed' })
observableResult.observe(current.arrayBuffers, { memoryType: 'arrayBuffers' })
observableResult.observe(current.external, { memoryType: 'external' })
observableResult.observe(current.rss, { memoryType: 'rss' })
})
}
private buildHandlesObserver () {
if (typeof (process as any)._getActiveHandles !== 'function') return
this.meter.createObservableGauge('nodejs_active_handles_total', {
description: 'Total number of active handles.'
}).addCallback(observableResult => {
const handles = (process as any)._getActiveHandles()
observableResult.observe(handles.length)
})
}
private buildGCObserver () {
const kinds = {
[constants.NODE_PERFORMANCE_GC_MAJOR]: 'major',
[constants.NODE_PERFORMANCE_GC_MINOR]: 'minor',
[constants.NODE_PERFORMANCE_GC_INCREMENTAL]: 'incremental',
[constants.NODE_PERFORMANCE_GC_WEAKCB]: 'weakcb'
}
const histogram = this.meter.createHistogram('nodejs_gc_duration_seconds', {
description: 'Garbage collection duration by kind, one of major, minor, incremental or weakcb'
})
const obs = new PerformanceObserver(list => {
const entry = list.getEntries()[0]
const kind = kinds[(entry.detail as NodeGCPerformanceDetail).kind]
// Convert duration from milliseconds to seconds
histogram.record(entry.duration / 1000, {
kind
})
})
obs.observe({ entryTypes: [ 'gc' ] })
}
private buildEventLoopLagObserver () {
const reportEventloopLag = (start: [ number, number ], observableResult: ObservableResult, res: () => void) => {
const delta = process.hrtime(start)
const nanosec = delta[0] * 1e9 + delta[1]
const seconds = nanosec / 1e9
observableResult.observe(seconds)
res()
}
this.meter.createObservableGauge('nodejs_eventloop_lag_seconds', {
description: 'Lag of event loop in seconds.'
}).addCallback(observableResult => {
return new Promise(res => {
const start = process.hrtime()
setImmediate(reportEventloopLag, start, observableResult, res)
})
})
}
private buildFileDescriptorsObserver () {
this.meter.createObservableGauge('process_open_fds', {
description: 'Number of open file descriptors.'
}).addCallback(async observableResult => {
try {
const fds = await readdir('/proc/self/fd')
observableResult.observe(fds.length - 1)
} catch (err) {
logger.debug('Cannot list file descriptors of current process for OpenTelemetry.', { err })
}
})
}
private buildLibUVActiveRequestsObserver () {
if (typeof (process as any)._getActiveRequests !== 'function') return
this.meter.createObservableGauge('nodejs_active_requests_total', {
description: 'Total number of active libuv requests.'
}).addCallback(observableResult => {
const requests = (process as any)._getActiveRequests()
observableResult.observe(requests.length)
})
}
private buildActiveResourcesObserver () {
if (typeof (process as any).getActiveResourcesInfo !== 'function') return
const grouped = this.meter.createObservableCounter('nodejs_active_resources', {
description: 'Number of active resources that are currently keeping the event loop alive, grouped by async resource type.'
})
const total = this.meter.createObservableCounter('nodejs_active_resources_total', {
description: 'Total number of active resources.'
})
this.meter.addBatchObservableCallback(observableResult => {
const resources = (process as any).getActiveResourcesInfo()
const data = {}
for (let i = 0; i < resources.length; i++) {
const resource = resources[i]
if (data[resource] === undefined) data[resource] = 0
data[resource] += 1
}
for (const type of Object.keys(data)) {
observableResult.observe(grouped, data[type], { type })
}
observableResult.observe(total, resources.length)
}, [ grouped, total ])
}
}
+85
ファイルの表示
@@ -0,0 +1,85 @@
import { Counter, Meter } from '@opentelemetry/api'
import { MVideoImmutable } from '@server/types/models/index.js'
import { PlaybackMetricCreate } from '@peertube/peertube-models'
export class PlaybackMetrics {
private errorsCounter: Counter
private resolutionChangesCounter: Counter
private downloadedBytesP2PCounter: Counter
private uploadedBytesP2PCounter: Counter
private downloadedBytesHTTPCounter: Counter
private peersP2PPeersGaugeBuffer: {
value: number
attributes: any
}[] = []
constructor (private readonly meter: Meter) {
}
buildCounters () {
this.errorsCounter = this.meter.createCounter('peertube_playback_errors_count', {
description: 'Errors collected from PeerTube player.'
})
this.resolutionChangesCounter = this.meter.createCounter('peertube_playback_resolution_changes_count', {
description: 'Resolution changes collected from PeerTube player.'
})
this.downloadedBytesHTTPCounter = this.meter.createCounter('peertube_playback_http_downloaded_bytes', {
description: 'Downloaded bytes with HTTP by PeerTube player.'
})
this.downloadedBytesP2PCounter = this.meter.createCounter('peertube_playback_p2p_downloaded_bytes', {
description: 'Downloaded bytes with P2P by PeerTube player.'
})
this.uploadedBytesP2PCounter = this.meter.createCounter('peertube_playback_p2p_uploaded_bytes', {
description: 'Uploaded bytes with P2P by PeerTube player.'
})
this.meter.createObservableGauge('peertube_playback_p2p_peers', {
description: 'Total P2P peers connected to the PeerTube player.'
}).addCallback(observableResult => {
for (const gauge of this.peersP2PPeersGaugeBuffer) {
observableResult.observe(gauge.value, gauge.attributes)
}
this.peersP2PPeersGaugeBuffer = []
})
}
observe (video: MVideoImmutable, metrics: PlaybackMetricCreate) {
const attributes = {
videoOrigin: video.remote
? 'remote'
: 'local',
playerMode: metrics.playerMode,
resolution: metrics.resolution + '',
fps: metrics.fps + '',
p2pEnabled: metrics.p2pEnabled,
videoUUID: video.uuid
}
this.errorsCounter.add(metrics.errors, attributes)
this.resolutionChangesCounter.add(metrics.resolutionChanges, attributes)
this.downloadedBytesHTTPCounter.add(metrics.downloadedBytesHTTP, attributes)
this.downloadedBytesP2PCounter.add(metrics.downloadedBytesP2P, attributes)
this.uploadedBytesP2PCounter.add(metrics.uploadedBytesP2P, attributes)
if (metrics.p2pPeers) {
this.peersP2PPeersGaugeBuffer.push({
value: metrics.p2pPeers,
attributes
})
}
}
}
+186
ファイルの表示
@@ -0,0 +1,186 @@
import memoizee from 'memoizee'
import { Meter } from '@opentelemetry/api'
import { MEMOIZE_TTL } from '@server/initializers/constants.js'
import { buildAvailableActivities } from '@server/lib/activitypub/activity.js'
import { StatsManager } from '@server/lib/stat-manager.js'
export class StatsObserversBuilder {
private readonly getInstanceStats = memoizee(() => {
return StatsManager.Instance.getStats()
}, { maxAge: MEMOIZE_TTL.GET_STATS_FOR_OPEN_TELEMETRY_METRICS })
constructor (private readonly meter: Meter) {
}
buildObservers () {
this.buildUserStatsObserver()
this.buildVideoStatsObserver()
this.buildCommentStatsObserver()
this.buildPlaylistStatsObserver()
this.buildChannelStatsObserver()
this.buildInstanceFollowsStatsObserver()
this.buildRedundancyStatsObserver()
this.buildActivityPubStatsObserver()
}
private buildUserStatsObserver () {
this.meter.createObservableGauge('peertube_users_total', {
description: 'Total users on the instance'
}).addCallback(async observableResult => {
const stats = await this.getInstanceStats()
observableResult.observe(stats.totalUsers)
})
this.meter.createObservableGauge('peertube_active_users_total', {
description: 'Total active users on the instance'
}).addCallback(async observableResult => {
const stats = await this.getInstanceStats()
observableResult.observe(stats.totalDailyActiveUsers, { activeInterval: 'daily' })
observableResult.observe(stats.totalWeeklyActiveUsers, { activeInterval: 'weekly' })
observableResult.observe(stats.totalMonthlyActiveUsers, { activeInterval: 'monthly' })
})
}
private buildChannelStatsObserver () {
this.meter.createObservableGauge('peertube_channels_total', {
description: 'Total channels on the instance'
}).addCallback(async observableResult => {
const stats = await this.getInstanceStats()
observableResult.observe(stats.totalLocalVideoChannels, { channelOrigin: 'local' })
})
this.meter.createObservableGauge('peertube_active_channels_total', {
description: 'Total active channels on the instance'
}).addCallback(async observableResult => {
const stats = await this.getInstanceStats()
observableResult.observe(stats.totalLocalDailyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'daily' })
observableResult.observe(stats.totalLocalWeeklyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'weekly' })
observableResult.observe(stats.totalLocalMonthlyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'monthly' })
})
}
private buildVideoStatsObserver () {
this.meter.createObservableGauge('peertube_videos_total', {
description: 'Total videos on the instance'
}).addCallback(async observableResult => {
const stats = await this.getInstanceStats()
observableResult.observe(stats.totalLocalVideos, { videoOrigin: 'local' })
observableResult.observe(stats.totalVideos - stats.totalLocalVideos, { videoOrigin: 'remote' })
})
this.meter.createObservableGauge('peertube_video_views_total', {
description: 'Total video views made on the instance'
}).addCallback(async observableResult => {
const stats = await this.getInstanceStats()
observableResult.observe(stats.totalLocalVideoViews, { viewOrigin: 'local' })
})
this.meter.createObservableGauge('peertube_video_bytes_total', {
description: 'Total bytes of videos'
}).addCallback(async observableResult => {
const stats = await this.getInstanceStats()
observableResult.observe(stats.totalLocalVideoFilesSize, { videoOrigin: 'local' })
})
}
private buildCommentStatsObserver () {
this.meter.createObservableGauge('peertube_comments_total', {
description: 'Total comments on the instance'
}).addCallback(async observableResult => {
const stats = await this.getInstanceStats()
observableResult.observe(stats.totalLocalVideoComments, { accountOrigin: 'local' })
})
}
private buildPlaylistStatsObserver () {
this.meter.createObservableGauge('peertube_playlists_total', {
description: 'Total playlists on the instance'
}).addCallback(async observableResult => {
const stats = await this.getInstanceStats()
observableResult.observe(stats.totalLocalPlaylists, { playlistOrigin: 'local' })
})
}
private buildInstanceFollowsStatsObserver () {
this.meter.createObservableGauge('peertube_instance_followers_total', {
description: 'Total followers of the instance'
}).addCallback(async observableResult => {
const stats = await this.getInstanceStats()
observableResult.observe(stats.totalInstanceFollowers)
})
this.meter.createObservableGauge('peertube_instance_following_total', {
description: 'Total following of the instance'
}).addCallback(async observableResult => {
const stats = await this.getInstanceStats()
observableResult.observe(stats.totalInstanceFollowing)
})
}
private buildRedundancyStatsObserver () {
this.meter.createObservableGauge('peertube_redundancy_used_bytes_total', {
description: 'Total redundancy used of the instance'
}).addCallback(async observableResult => {
const stats = await this.getInstanceStats()
for (const r of stats.videosRedundancy) {
observableResult.observe(r.totalUsed, { strategy: r.strategy })
}
})
this.meter.createObservableGauge('peertube_redundancy_available_bytes_total', {
description: 'Total redundancy available of the instance'
}).addCallback(async observableResult => {
const stats = await this.getInstanceStats()
for (const r of stats.videosRedundancy) {
observableResult.observe(r.totalSize, { strategy: r.strategy })
}
})
}
private buildActivityPubStatsObserver () {
const availableActivities = buildAvailableActivities()
this.meter.createObservableGauge('peertube_ap_inbox_success_total', {
description: 'Total inbox messages processed with success'
}).addCallback(async observableResult => {
const stats = await this.getInstanceStats()
for (const type of availableActivities) {
observableResult.observe(stats[`totalActivityPub${type}MessagesSuccesses`], { activityType: type })
}
})
this.meter.createObservableGauge('peertube_ap_inbox_error_total', {
description: 'Total inbox messages processed with error'
}).addCallback(async observableResult => {
const stats = await this.getInstanceStats()
for (const type of availableActivities) {
observableResult.observe(stats[`totalActivityPub${type}MessagesErrors`], { activityType: type })
}
})
this.meter.createObservableGauge('peertube_ap_inbox_waiting_total', {
description: 'Total inbox messages waiting for being processed'
}).addCallback(async observableResult => {
const stats = await this.getInstanceStats()
observableResult.observe(stats.totalActivityPubMessagesWaiting)
})
}
}
+24
ファイルの表示
@@ -0,0 +1,24 @@
import { Meter } from '@opentelemetry/api'
import { VideoScope, ViewerScope } from '@server/lib/views/shared/index.js'
import { VideoViewsManager } from '@server/lib/views/video-views-manager.js'
export class ViewersObserversBuilder {
constructor (private readonly meter: Meter) {
}
buildObservers () {
this.meter.createObservableGauge('peertube_viewers_total', {
description: 'Total viewers on the instance'
}).addCallback(observableResult => {
for (const viewerScope of [ 'local', 'remote' ] as ViewerScope[]) {
for (const videoScope of [ 'local', 'remote' ] as VideoScope[]) {
const result = VideoViewsManager.Instance.getTotalViewers({ viewerScope, videoScope })
observableResult.observe(result, { viewerOrigin: viewerScope, videoOrigin: videoScope })
}
}
})
}
}
+32
ファイルの表示
@@ -0,0 +1,32 @@
import { Meter } from '@opentelemetry/api'
import { getWorkersStats } from '@server/lib/worker/parent-process.js'
export class WorkerThreadsObserversBuilder {
constructor (private readonly meter: Meter) {
}
buildObservers () {
this.meter.createObservableGauge('peertube_worker_thread_queue_total', {
description: 'Total tasks waiting for a PeerTube worker thread'
}).addCallback(observableResult => {
const stats = getWorkersStats()
for (const stat of stats) {
observableResult.observe(stat.queueSize, { state: 'waiting', workerThread: stat.label })
}
})
this.meter.createObservableGauge('peertube_worker_thread_completed_total', {
description: 'Total tasks completed in PeerTube worker threads'
}).addCallback(observableResult => {
const stats = getWorkersStats()
for (const stat of stats) {
observableResult.observe(stat.completed, { workerThread: stat.label })
}
})
}
}