btrc-hub/frontend/src/lib/sitemap.server.ts

import { generatePath } from 'react-router-dom'

import { pageRoutes } from '@/App'
import { apiGet, isApiError } from '@/lib/api'
import { parseHelpEdition } from '@/lib/help'
import { fetchLocales, isLocaleCode, localeLanguage } from '@/lib/locales'
import { withRenderContext } from '@/lib/renderContext.server'
import { isCountsPath } from '@/lib/routeLocales'

import type { LocaleCode } from '@/lib/locales'
import type { PageRoute, SitemapSource } from '@/lib/routeLocales'

type SitemapRoute = { path: string; source: SitemapSource }
type SitemapResponse = { status: number; body: string }
type ResourceCounts = {
    resources: { resource: SitemapSource; count: number }[]
    pageSize: number }

const PAGE_SIZE = 1_000


const sitemapRoutes = (locale: LocaleCode): SitemapRoute[] => {
  const collect = (routes: PageRoute[], parent = ''): SitemapRoute[] =>
      routes.flatMap (route => {
	const handle = route.handle
	if (handle?.robots === 'noindex' || handle?.authentication != null
	    || handle?.redirectTo != null || handle?.notFound
	    || (handle?.locale != null && handle.locale !== locale))
	  return []
	const path = [parent, route.path].filter (Boolean).join ('/')
	if (isCountsPath (`/${path}`)) return []
	return [
	    ...(handle?.sitemap ?? []).map (source => ({ path, source })),
	    ...collect (route.children ?? [], path)]
      })
  return collect (pageRoutes (null, () => { }))
}


const helpIdentifiers = async (locale: LocaleCode): Promise<string[]> => {
  const { default: articles } = await import ('virtual:help/articles')
  const language = localeLanguage (locale)
  return Object.values (articles).filter (article => {
    const edition = article.languages[language]
    return edition != null && edition.publishedEditions.includes (edition.currentEdition)
	&& parseHelpEdition (edition.currentEdition) != null
  }).map (article => article.identifier).sort ()
}


const escapeXML = (value: string): string =>
    value.replace (/&/g, '&amp;').replace (/</g, '&lt;').replace (/>/g, '&gt;')
	.replace (/"/g, '&quot;').replace (/'/g, '&apos;')


const sitemapXML = (urls: string[], index = false): string => {
  if (urls.length > 50_000)
    throw new Error ('Sitemap URL limit exceeded')
  const root = index ? 'sitemapindex' : 'urlset'
  const item = index ? 'sitemap' : 'url'
  return '<?xml version="1.0" encoding="UTF-8"?>\n'
      + `<${root} xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n`
      + urls.map (url => `  <${item}><loc>${escapeXML (url)}</loc></${item}>`).join ('\n')
      + `\n</${root}>\n`
}


export const renderSitemap = async (
    address: string,
    origin: string,
    apiBaseURL: string,
): Promise<SitemapResponse> =>
    withRenderContext ({ pathname: address, origin, apiBaseURL }, async () => {
      const url = new URL (address, origin)
      const index = url.pathname === '/sitemap.xml'
      const matched = url.pathname.match (/^\/sitemap-(.+)-([a-z_]+)-([1-9]\d*)\.xml$/)
      if (!index && matched == null)
	return { status: 404, body: '' }

      // Sitemap requests are always public; incoming user cookies are not forwarded.
      const available = await fetchLocales ()
      const locales = available.locales.map (entry => entry.code).filter (isLocaleCode)
      if (index)
      {
	const urls: string[] = []
	for (const locale of locales)
	{
	  const sources = new Set (sitemapRoutes (locale).map (route => route.source))
	  const counts = await apiGet<ResourceCounts> ('/sitemap_entries', { locale })
	  for (const source of sources)
	  {
	    const count =
		source === 'static'
		? 1
		: (source === 'help'
		   ? (await helpIdentifiers (locale)).length
		   : counts.resources.find (entry => entry.resource === source)?.count ?? 0)
	    const pageSize = source === 'help' ? PAGE_SIZE : counts.pageSize
	    if (!Number.isSafeInteger (pageSize) || pageSize <= 0
		|| !Number.isSafeInteger (count) || count < 0)
	      throw new Error ('Invalid sitemap pagination')
	    const pages = Math.ceil (count / pageSize)
	    if (urls.length + pages > 50_000)
	      throw new Error ('Sitemap index limit exceeded')
	    for (let page = 1; page <= pages; ++page)
	      urls.push (new URL (`/sitemap-${locale}-${source}-${page}.xml`, origin).href)
	  }
	}
	return { status: 200, body: sitemapXML (urls, true) }
      }

      const [, locale, source, pageValue] = matched!
      const page = Number (pageValue)
      if (!isLocaleCode (locale) || !locales.includes (locale)
	  || !Number.isSafeInteger (page))
	return { status: 404, body: '' }
      const routes = sitemapRoutes (locale).filter (route => route.source === source)
      if (routes.length === 0)
	return { status: 404, body: '' }

      const parameters: Record<string, string>[] = []
      if (source === 'static')
      {
	if (page !== 1)
	  return { status: 404, body: '' }
	parameters.push ({ })
      }
      else if (source === 'help')
      {
	const identifiers = await helpIdentifiers (locale)
	parameters.push (...identifiers.slice ((page - 1) * PAGE_SIZE, page * PAGE_SIZE)
	    .map (identifier => ({ identifier: encodeURIComponent (identifier) })))
      }
      else
      {
	try
	{
	  const { ids } = await apiGet<{ ids: number[] }> (
	      `/sitemap_entries/${source}`, { locale, params: { page } })
	  parameters.push (...ids.map (id => ({ id: String (id), title: String (id) })))
	}
	catch (error)
	{
	  if (isApiError (error) && error.response?.status === 404)
	    return { status: 404, body: '' }
	  throw error
	}
      }
      if (parameters.length === 0)
	return { status: 404, body: '' }
      const urls = routes.flatMap (route => parameters.map (params =>
	  new URL (generatePath (`/${locale}/${route.path}`, params), origin).href))
      return { status: 200, body: sitemapXML ([...new Set (urls)]) }
    })