yovision/Sense/scripts/t014-capacity.ps1

[CmdletBinding()]
param(
    [string]$PgRoot = 'D:\pgsql17',
    [string]$RuntimeRoot = (Join-Path ([IO.Path]::GetTempPath()) 'yovision-t014'),
    [ValidateRange(1, 1440)]
    [int]$ObservationMinutes = 30,
    [string]$OutputPath = '',
    [switch]$PreflightOnly
)

$ErrorActionPreference = 'Stop'
$ProgressPreference = 'SilentlyContinue'
Set-StrictMode -Version Latest

$sourceCount = 16
$faultIndexes = @(5, 6, 7, 8)
$sampleSeconds = 10
$mediaMTXVersion = 'v1.19.3'
$mediaMTXSHA256 = '5d82148d1032a6a190d9909a2997d9989457aaadf49af87dd02cd4512d31bebe'
$mediaMTXExecutableSHA256 = '1cda85249312cb9463f9f94c5a712b9f160c9af3fd9490f0d4723911d7880e05'
$migrationNames = @(
    '002_bell.sql',
    '003_sense.sql',
    '004_privileges.sql',
    '005_area_policy.sql',
    '006_device_operation_outbox.sql',
    '007_privileges_area_audit.sql',
    '008_control_api.sql',
    '009_privileges_control_api.sql',
    '010_reconcile_safety.sql',
    '011_privileges_reconcile_safety.sql'
)

function ConvertTo-Base64Url([byte[]]$Bytes) {
    return [Convert]::ToBase64String($Bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_')
}

function New-RandomBytes([int]$Count) {
    $bytes = [byte[]]::new($Count)
    [Security.Cryptography.RandomNumberGenerator]::Fill($bytes)
    return $bytes
}

function New-SecretToken {
    return ConvertTo-Base64Url (New-RandomBytes 32)
}

function Get-SHA256Hex([string]$Value) {
    $bytes = [Text.Encoding]::UTF8.GetBytes($Value)
    $digest = [Security.Cryptography.SHA256]::HashData($bytes)
    return ([Convert]::ToHexString($digest)).ToLowerInvariant()
}

function Get-FreeTcpPort([Collections.Generic.HashSet[int]]$Reserved) {
    while ($true) {
        $listener = [Net.Sockets.TcpListener]::new([Net.IPAddress]::Loopback, 0)
        $listener.Start()
        $port = ([Net.IPEndPoint]$listener.LocalEndpoint).Port
        $listener.Stop()
        if ($Reserved.Add($port)) {
            return $port
        }
    }
}

function Wait-Port([int]$Port, [int]$TimeoutSeconds = 30) {
    $watch = [Diagnostics.Stopwatch]::StartNew()
    while ($watch.Elapsed.TotalSeconds -lt $TimeoutSeconds) {
        $client = [Net.Sockets.TcpClient]::new()
        try {
            $task = $client.ConnectAsync('127.0.0.1', $Port)
            if ($task.Wait(500) -and $client.Connected) {
                return
            }
        }
        catch {
        }
        finally {
            $client.Dispose()
        }
        Start-Sleep -Milliseconds 250
    }
    throw "local process did not listen before timeout"
}

function Start-ManagedProcess(
    [string]$Name,
    [string]$FilePath,
    [string[]]$Arguments,
    [hashtable]$Environment = @{}
) {
    $start = [Diagnostics.ProcessStartInfo]::new()
    $start.FileName = $FilePath
    $start.WorkingDirectory = $script:session
    $start.UseShellExecute = $false
    $start.CreateNoWindow = $true
    $start.RedirectStandardOutput = $true
    $start.RedirectStandardError = $true
    foreach ($argument in $Arguments) {
        $start.ArgumentList.Add($argument)
    }
    foreach ($entry in $Environment.GetEnumerator()) {
        $start.Environment[$entry.Key] = [string]$entry.Value
    }
    $process = [Diagnostics.Process]::new()
    $process.StartInfo = $start
    if (-not $process.Start()) {
        throw "failed to start required process: $Name"
    }
    return [pscustomobject]@{
        Name = $Name
        Process = $process
        Stdout = $process.StandardOutput.ReadToEndAsync()
        Stderr = $process.StandardError.ReadToEndAsync()
    }
}

function Stop-ManagedProcess($Managed) {
    if ($null -eq $Managed -or $null -eq $Managed.Process) {
        return
    }
    if (-not $Managed.Process.HasExited) {
        $Managed.Process.Kill()
        $Managed.Process.WaitForExit(10000) | Out-Null
    }
}

function Stop-AllManagedProcesses($Items) {
    $processes = @($Items | Where-Object { $null -ne $_ -and $null -ne $_.Process })
    foreach ($item in $processes) {
        try {
            if (-not $item.Process.HasExited) {
                $item.Process.Kill()
            }
        }
        catch {
            if (-not $item.Process.HasExited) { throw }
        }
    }
    $cleanupWatch = [Diagnostics.Stopwatch]::StartNew()
    while ($cleanupWatch.Elapsed.TotalSeconds -lt 10 -and
        @($processes | Where-Object { -not $_.Process.HasExited }).Count -gt 0) {
        Start-Sleep -Milliseconds 100
    }
    $remaining = @($processes | Where-Object { -not $_.Process.HasExited })
    if ($remaining.Count -gt 0) {
        Write-Warning "managed processes did not exit within the shared cleanup deadline: $($remaining.Count)"
    }
    foreach ($item in $processes) {
        if ($item.Process.HasExited) {
            $item.Stdout.Wait(2000) | Out-Null
            $item.Stderr.Wait(2000) | Out-Null
            $item.Process.Dispose()
        }
    }
}

function Remove-SessionDirectory([string]$Path) {
    Get-ChildItem -LiteralPath $Path -Recurse -Force -ErrorAction SilentlyContinue |
        ForEach-Object { $_.Attributes = 'Normal' }
    (Get-Item -LiteralPath $Path -Force).Attributes = 'Directory'
    $deleteWatch = [Diagnostics.Stopwatch]::StartNew()
    while (Test-Path -LiteralPath $Path) {
        try {
            Remove-Item -LiteralPath $Path -Recurse -Force -ErrorAction Stop
        }
        catch {
            if ($deleteWatch.Elapsed.TotalSeconds -ge 10) { throw }
            Start-Sleep -Milliseconds 200
        }
    }
}

function Assert-Alive($Managed) {
    if ($null -eq $Managed -or $Managed.Process.HasExited) {
        $code = if ($null -eq $Managed) { 'not-started' } else { $Managed.Process.ExitCode }
        throw "required process exited: $($Managed.Name), code=$code"
    }
}

function Invoke-Checked {
    param(
        [Parameter(Mandatory)] [string]$FilePath,
        [Parameter(ValueFromRemainingArguments)] [string[]]$Arguments
    )
    & $FilePath @Arguments 2>&1 | Out-Null
    if ($LASTEXITCODE -ne 0) {
        throw "required command failed with exit code $LASTEXITCODE"
    }
}

function Invoke-ControlRequest(
    [string]$Method,
    [string]$Path,
    [int[]]$ExpectedStatus,
    $Body = $null,
    [hashtable]$ExtraHeaders = @{}
) {
    $headers = @{
        Authorization = "Bearer $script:controlToken"
        Accept = 'application/json'
    }
    foreach ($entry in $ExtraHeaders.GetEnumerator()) {
        $headers[$entry.Key] = $entry.Value
    }
    $parameters = @{
        Method = $Method
        Uri = "http://127.0.0.1:$script:senseHTTPPort$Path"
        Headers = $headers
        TimeoutSec = 15
        NoProxy = $true
        SkipHttpErrorCheck = $true
    }
    if ($null -ne $Body) {
        $parameters.ContentType = 'application/json'
        $parameters.Body = ($Body | ConvertTo-Json -Depth 10 -Compress)
    }
    $response = Invoke-WebRequest @parameters
    $statusCode = [int]$response.StatusCode
    $statusMatches = $false
    foreach ($expected in $ExpectedStatus) {
        if ([int]$expected -eq $statusCode) {
            $statusMatches = $true
            break
        }
    }
    if (-not $statusMatches) {
        $code = 'unknown'
        try {
            $code = ((Get-ResponseText $response) | ConvertFrom-Json).code
        }
        catch {
        }
        throw "Control API returned unexpected status $statusCode, expected=$($ExpectedStatus -join ','), code=$code"
    }
    return $response
}

function Get-ResponseText($Response) {
    if ($Response.Content -is [byte[]]) {
        return [Text.Encoding]::UTF8.GetString($Response.Content)
    }
    return [string]$Response.Content
}

function Convert-ResponseJSON($Response) {
    return ((Get-ResponseText $Response) | ConvertFrom-Json)
}

function Get-ResponseETag($Response) {
    $value = [string]$Response.Headers.ETag
    if ([string]::IsNullOrWhiteSpace($value)) {
        throw 'Control API response did not contain ETag'
    }
    return $value
}

function Get-DevicePage {
    $response = Invoke-ControlRequest 'GET' '/api/v1/sites/site-t014/devices?limit=100' @(200)
    return Convert-ResponseJSON $response
}

function Get-DeviceEntries([string[]]$DeviceIDs) {
    $entries = [Collections.Generic.List[object]]::new()
    foreach ($id in $DeviceIDs) {
        $response = Invoke-ControlRequest 'GET' "/api/v1/sites/site-t014/devices/$id" @(200)
        $entries.Add([pscustomobject]@{
            DeviceID = $id
            ETag = Get-ResponseETag $response
        })
    }
    return @($entries)
}

function Invoke-BatchDesiredState(
    [string]$DesiredState,
    [object[]]$Entries,
    [string]$KeySuffix
) {
    $items = @($Entries | ForEach-Object {
        [ordered]@{
            device_id = $_.DeviceID
            etag = $_.ETag
            desired_state = $DesiredState
        }
    })
    $watch = [Diagnostics.Stopwatch]::StartNew()
    $response = Invoke-ControlRequest 'POST' '/api/v1/sites/site-t014/devices:batchDesiredState' @(202) ([ordered]@{
        items = $items
        reason = "T-014 laboratory $DesiredState baseline"
    }) @{ 'Idempotency-Key' = "t014-batch-$KeySuffix-$script:runNonce" }
    $operation = Convert-ResponseJSON $response
    $results = @($operation.results)
    if ($operation.status -ne 'succeeded' -or $results.Count -ne $sourceCount -or
        @($results | Where-Object { $_.status -ne 'succeeded' }).Count -ne 0) {
        throw "batch desired-state operation was not fully successful"
    }
    return [ordered]@{
        status = [string]$operation.status
        item_count = $results.Count
        request_seconds = [Math]::Round($watch.Elapsed.TotalSeconds, 3)
    }
}

function Get-MediaInventory([int]$APIPort) {
    $config = Invoke-RestMethod -Method Get -Uri "http://127.0.0.1:$APIPort/v3/config/paths/list?page=0&itemsPerPage=100" -TimeoutSec 5 -NoProxy
    $runtime = Invoke-RestMethod -Method Get -Uri "http://127.0.0.1:$APIPort/v3/paths/list?page=0&itemsPerPage=100" -TimeoutSec 5 -NoProxy
    $items = @($runtime.items)
    [UInt64]$inboundBytes = 0
    [UInt64]$frameErrors = 0
    foreach ($item in $items) {
        if ($null -ne $item.inboundBytes) {
            $inboundBytes += [UInt64]$item.inboundBytes
        }
        if ($null -ne $item.inboundFramesInError) {
            $frameErrors += [UInt64]$item.inboundFramesInError
        }
    }
    return [pscustomobject]@{
        Configured = [int]$config.itemCount
        Runtime = [int]$runtime.itemCount
        Online = @($items | Where-Object { $_.online -eq $true -and $_.available -eq $true }).Count
        InboundBytes = $inboundBytes
        FrameErrors = $frameErrors
    }
}

function Get-SenseMetric([string]$Name, [string]$Labels) {
    $response = Invoke-WebRequest -Method Get -Uri "http://127.0.0.1:$script:senseHTTPPort/metrics" -TimeoutSec 5 -NoProxy
    $prefix = [Regex]::Escape($Name + $Labels)
    $match = [Regex]::Match((Get-ResponseText $response), "(?m)^$prefix\s+([-+0-9.eE]+)$")
    if (-not $match.Success) {
        throw "required Sense metric was not found: $Name"
    }
    return [double]::Parse($match.Groups[1].Value, [Globalization.CultureInfo]::InvariantCulture)
}

function Wait-SenseHealth($Managed, [int]$TimeoutSeconds = 45) {
    $watch = [Diagnostics.Stopwatch]::StartNew()
    while ($watch.Elapsed.TotalSeconds -lt $TimeoutSeconds) {
        Assert-Alive $Managed
        try {
            $ready = Invoke-RestMethod -Method Get -Uri "http://127.0.0.1:$script:senseHTTPPort/readyz" -TimeoutSec 2 -NoProxy
            if ($ready.status -eq 'ready') {
                return
            }
        }
        catch {
        }
        Start-Sleep -Milliseconds 500
    }
    throw 'Sense readiness endpoint did not become ready'
}

function Wait-SourcePublishers([int]$TimeoutSeconds = 90) {
    $watch = [Diagnostics.Stopwatch]::StartNew()
    while ($watch.Elapsed.TotalSeconds -lt $TimeoutSeconds) {
        foreach ($publisher in $script:publishers.Values) {
            Assert-Alive $publisher
        }
        try {
            $inventory = Get-MediaInventory $script:sourceAPIPort
            if ($inventory.Runtime -eq $sourceCount -and $inventory.Online -eq $sourceCount) {
                return [Math]::Round($watch.Elapsed.TotalSeconds, 1)
            }
        }
        catch {
        }
        Start-Sleep -Seconds 1
    }
    throw 'independent source publishers did not all become online'
}

function Wait-Convergence([ValidateSet('enabled', 'disabled')] [string]$DesiredState, [int]$TimeoutSeconds = 300) {
    $watch = [Diagnostics.Stopwatch]::StartNew()
    $last = 'no observation'
    while ($watch.Elapsed.TotalSeconds -lt $TimeoutSeconds) {
        try {
            $page = Get-DevicePage
            $devices = @($page.items)
            $inventory = Get-MediaInventory $script:productionAPIPort
            $totalMetric = Get-SenseMetric 'sense_reconcile_devices' '{state="total"}'
            $unconvergedMetric = Get-SenseMetric 'sense_reconcile_devices' '{state="unconverged"}'
            if ($DesiredState -eq 'enabled') {
                $stateOK = $devices.Count -eq $sourceCount -and
                    @($devices | Where-Object {
                        $_.desired_state -ne 'enabled' -or $_.actual_state -ne 'online' -or -not $_.converged
                    }).Count -eq 0
                $mediaOK = $inventory.Configured -eq $sourceCount -and
                    $inventory.Runtime -eq $sourceCount -and $inventory.Online -eq $sourceCount
                $metricsOK = $totalMetric -eq $sourceCount -and $unconvergedMetric -eq 0
            }
            else {
                $stateOK = $devices.Count -eq $sourceCount -and
                    @($devices | Where-Object {
                        $_.desired_state -ne 'disabled' -or $_.actual_state -ne 'offline' -or -not $_.converged
                    }).Count -eq 0
                $mediaOK = $inventory.Configured -eq 0 -and $inventory.Runtime -eq 0
                $metricsOK = $totalMetric -eq 0 -and $unconvergedMetric -eq 0
            }
            $last = "devices=$($devices.Count), configured=$($inventory.Configured), runtime=$($inventory.Runtime), online=$($inventory.Online), unconverged=$unconvergedMetric"
            if ($stateOK -and $mediaOK -and $metricsOK) {
                return [Math]::Round($watch.Elapsed.TotalSeconds, 1)
            }
        }
        catch {
            $last = $_.Exception.Message
        }
        Start-Sleep -Seconds 1
    }
    throw "devices did not reach $DesiredState convergence: $last"
}

function Wait-PartialFault([int]$TimeoutSeconds = 120) {
    $watch = [Diagnostics.Stopwatch]::StartNew()
    while ($watch.Elapsed.TotalSeconds -lt $TimeoutSeconds) {
        try {
            $page = Get-DevicePage
            $devices = @($page.items)
            $online = @($devices | Where-Object { $_.actual_state -eq 'online' }).Count
            $notOnline = $devices.Count - $online
            if ($devices.Count -eq $sourceCount -and $online -eq ($sourceCount - $faultIndexes.Count) -and
                $notOnline -eq $faultIndexes.Count) {
                return [ordered]@{
                    detection_seconds = [Math]::Round($watch.Elapsed.TotalSeconds, 1)
                    affected = $notOnline
                    unaffected_online = $online
                }
            }
        }
        catch {
        }
        Start-Sleep -Seconds 1
    }
    throw 'four-source fault was not isolated to exactly four devices'
}

function Start-Publisher([int]$Index) {
    $name = 'synthetic-{0:D2}' -f $Index
    return Start-ManagedProcess "publisher-$Index" $script:ffmpeg @(
        '-hide_banner', '-loglevel', 'warning', '-re', '-stream_loop', '-1',
        '-i', $script:fixturePath, '-map', '0:v:0', '-c', 'copy', '-an',
        '-f', 'rtsp', '-rtsp_transport', 'tcp',
        "rtsp://127.0.0.1:$script:sourceRTSPPort/$name"
    )
}

function Get-ProcessResourceSample($Managed, [double]$PreviousCPU, [double]$ElapsedSeconds) {
    Assert-Alive $Managed
    $Managed.Process.Refresh()
    $totalCPU = $Managed.Process.TotalProcessorTime.TotalSeconds
    $normalizedCPU = 0.0
    if ($ElapsedSeconds -gt 0) {
        $normalizedCPU = (($totalCPU - $PreviousCPU) / $ElapsedSeconds / [Environment]::ProcessorCount) * 100
    }
    return [pscustomobject]@{
        TotalCPU = $totalCPU
        CPUPercent = [Math]::Max(0, $normalizedCPU)
        WorkingSetMiB = $Managed.Process.WorkingSet64 / 1MB
        PrivateMiB = $Managed.Process.PrivateMemorySize64 / 1MB
        Handles = $Managed.Process.HandleCount
    }
}

function Get-Average([Collections.Generic.List[double]]$Values) {
    if ($Values.Count -eq 0) { return 0.0 }
    return ($Values | Measure-Object -Average).Average
}

function Get-Maximum([Collections.Generic.List[double]]$Values) {
    if ($Values.Count -eq 0) { return 0.0 }
    return ($Values | Measure-Object -Maximum).Maximum
}

function Get-PostgresScalar([string]$Query) {
    $value = & $script:psql -X -A -t -v ON_ERROR_STOP=1 -d $script:adminDatabaseDSN -c $Query
    if ($LASTEXITCODE -ne 0) {
        throw 'PostgreSQL observation query failed'
    }
    return (($value | Out-String).Trim())
}

function Write-Result([string]$JSON) {
    if (-not [string]::IsNullOrWhiteSpace($OutputPath)) {
        $resolvedOutput = [IO.Path]::GetFullPath($OutputPath)
        $repoPrefix = $script:repoRoot.TrimEnd('\') + '\'
        if ($resolvedOutput.StartsWith($repoPrefix, [StringComparison]::OrdinalIgnoreCase)) {
            throw 'capacity result output must stay outside the repository'
        }
        $parent = Split-Path -Parent $resolvedOutput
        if (-not (Test-Path -LiteralPath $parent -PathType Container)) {
            throw 'capacity result output directory does not exist'
        }
        [IO.File]::WriteAllText($resolvedOutput, $JSON + [Environment]::NewLine, [Text.UTF8Encoding]::new($false))
    }
    Write-Output $JSON
}

$script:repoRoot = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot '..\..')).Path
$pgRootResolved = (Resolve-Path -LiteralPath $PgRoot).Path
$pgBin = Join-Path $pgRootResolved 'bin'
$script:initdb = Join-Path $pgBin 'initdb.exe'
$script:pgCtl = Join-Path $pgBin 'pg_ctl.exe'
$script:psql = Join-Path $pgBin 'psql.exe'
$script:createdb = Join-Path $pgBin 'createdb.exe'
$postgres = Join-Path $pgBin 'postgres.exe'
foreach ($required in @($script:initdb, $script:pgCtl, $script:psql, $script:createdb, $postgres)) {
    if (-not (Test-Path -LiteralPath $required -PathType Leaf)) {
        throw 'PostgreSQL executable is missing under the selected PgRoot'
    }
}
$postgresVersion = (& $postgres --version 2>&1 | Out-String).Trim()
if ($LASTEXITCODE -ne 0 -or $postgresVersion -notmatch 'PostgreSQL\) 17\.10$') {
    throw 'T-014 requires the frozen PostgreSQL 17.10 binaries'
}
$script:ffmpeg = (Get-Command ffmpeg -ErrorAction Stop).Source
$ffmpegVersion = (& $script:ffmpeg -version 2>&1 | Select-Object -First 1)
if ($ffmpegVersion -notmatch '^ffmpeg version 8\.1\.2') {
    throw 'T-014 requires the frozen FFmpeg 8.1.2 build'
}
$goVersion = (& go version 2>&1 | Out-String).Trim()
if ($LASTEXITCODE -ne 0 -or $goVersion -notmatch '^go version go1\.23\.') {
    throw 'T-014 requires the frozen Go 1.23 toolchain'
}

$systemTemp = [IO.Path]::GetFullPath([IO.Path]::GetTempPath()).TrimEnd('\')
$runtimeResolved = [IO.Path]::GetFullPath($RuntimeRoot).TrimEnd('\')
$runtimePrefix = $systemTemp + '\'
if (-not $runtimeResolved.StartsWith($runtimePrefix, [StringComparison]::OrdinalIgnoreCase) -or
    [IO.Path]::GetFileName($runtimeResolved) -notlike 'yovision-t014*') {
    throw 'RuntimeRoot must be a yovision-t014 directory under the system temporary directory'
}

$mediaDirectory = Join-Path $runtimeResolved 'mediamtx-v1.19.3'
$mediaMTX = Join-Path $mediaDirectory 'mediamtx.exe'
if ($PreflightOnly) {
    $memoryGiB = 0.0
    try { $memoryGiB = (Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory / 1GB } catch {}
    Write-Result (([ordered]@{
        success = $true
        preflight_only = $true
        source_count = $sourceCount
        observation_minutes = $ObservationMinutes
        formal_eligible = $ObservationMinutes -ge 30
        mediamtx_cached = Test-Path -LiteralPath $mediaMTX -PathType Leaf
        postgres_version = $postgresVersion
        ffmpeg_version = [string]$ffmpegVersion
        go_version = $goVersion
        logical_processors = [Environment]::ProcessorCount
        total_memory_gib = [Math]::Round($memoryGiB, 2)
    } | ConvertTo-Json -Depth 5))
    exit 0
}

New-Item -ItemType Directory -Path $runtimeResolved -Force | Out-Null
if (-not (Test-Path -LiteralPath $mediaMTX -PathType Leaf)) {
    $zip = Join-Path $runtimeResolved 'mediamtx_v1.19.3_windows_amd64.zip'
    Invoke-WebRequest 'https://github.com/bluenviron/mediamtx/releases/download/v1.19.3/mediamtx_v1.19.3_windows_amd64.zip' -OutFile $zip
    $actualHash = (Get-FileHash -LiteralPath $zip -Algorithm SHA256).Hash.ToLowerInvariant()
    if ($actualHash -ne $mediaMTXSHA256) {
        throw 'MediaMTX checksum mismatch'
    }
    New-Item -ItemType Directory -Path $mediaDirectory -Force | Out-Null
    Expand-Archive -LiteralPath $zip -DestinationPath $mediaDirectory -Force
}
if ((Get-FileHash -LiteralPath $mediaMTX -Algorithm SHA256).Hash.ToLowerInvariant() -ne $mediaMTXExecutableSHA256) {
    throw 'MediaMTX executable checksum mismatch'
}

$script:runNonce = [Guid]::NewGuid().ToString('N')
$script:session = Join-Path $runtimeResolved ("session-$script:runNonce")
New-Item -ItemType Directory -Path $script:session | Out-Null
$reservedPorts = [Collections.Generic.HashSet[int]]::new()
$script:sourceRTSPPort = Get-FreeTcpPort $reservedPorts
$script:sourceAPIPort = Get-FreeTcpPort $reservedPorts
$productionRTSPPort = Get-FreeTcpPort $reservedPorts
$script:productionAPIPort = Get-FreeTcpPort $reservedPorts
$productionMetricsPort = Get-FreeTcpPort $reservedPorts
$script:senseHTTPPort = Get-FreeTcpPort $reservedPorts
$postgresPort = Get-FreeTcpPort $reservedPorts

$sourceConfig = Join-Path $script:session 'mediamtx-source.yml'
$productionConfig = Join-Path $script:session 'mediamtx-production.yml'
$sourceLines = [Collections.Generic.List[string]]::new()
@(
    'logLevel: warn',
    "rtspAddress: 127.0.0.1:$script:sourceRTSPPort",
    'rtspTransports: [tcp]',
    'api: true',
    "apiAddress: 127.0.0.1:$script:sourceAPIPort",
    'metrics: false', 'rtmp: false', 'hls: false', 'webrtc: false', 'srt: false', 'moq: false',
    'paths:'
) | ForEach-Object { $sourceLines.Add($_) }
foreach ($index in 1..$sourceCount) {
    $sourceLines.Add(('  synthetic-{0:D2}:' -f $index))
    $sourceLines.Add('    source: publisher')
}
[IO.File]::WriteAllLines($sourceConfig, $sourceLines, [Text.UTF8Encoding]::new($false))
[IO.File]::WriteAllLines($productionConfig, @(
    'logLevel: warn',
    "rtspAddress: 127.0.0.1:$productionRTSPPort",
    'rtspTransports: [tcp]',
    'api: true',
    "apiAddress: 127.0.0.1:$script:productionAPIPort",
    'metrics: true',
    "metricsAddress: 127.0.0.1:$productionMetricsPort",
    'rtmp: false', 'hls: false', 'webrtc: false', 'srt: false', 'moq: false',
    'paths: {}'
), [Text.UTF8Encoding]::new($false))

$script:fixturePath = Join-Path $script:session 'fixture.mp4'
$senseBinary = Join-Path $script:session 'sense-api.exe'
$authPath = Join-Path $script:session 'control-auth.json'
$cursorPath = Join-Path $script:session 'cursor.key'
$pgData = Join-Path $script:session 'pgdata'
$pgLog = Join-Path $script:session 'postgres.log'
$databaseName = 'yovision_t014'
$adminRootDSN = "postgres://postgres@127.0.0.1:$postgresPort/postgres?sslmode=disable"
$script:adminDatabaseDSN = "postgres://postgres@127.0.0.1:$postgresPort/${databaseName}?sslmode=disable"
$senseDSN = "postgres://yovision_t014_sense@127.0.0.1:$postgresPort/${databaseName}?sslmode=disable"
$existing5432 = @(
    Get-NetTCPConnection -State Listen -LocalPort 5432 -ErrorAction SilentlyContinue |
        Select-Object -ExpandProperty OwningProcess -Unique | Sort-Object
)

$managed = [Collections.Generic.List[object]]::new()
$script:publishers = @{}
$postgresStarted = $false
$postgresStopped = $false
$success = $false
$stage = 'preparing fixtures'
$failure = $null
$resultJSON = $null
try {
    Invoke-Checked $script:ffmpeg '-hide_banner' '-loglevel' 'error' '-f' 'lavfi' '-i' 'testsrc2=size=640x360:rate=10' '-t' '30' '-c:v' 'libx264' '-preset' 'ultrafast' '-tune' 'zerolatency' '-pix_fmt' 'yuv420p' '-g' '10' '-an' '-movflags' '+faststart' '-y' $script:fixturePath
    Invoke-Checked 'go' '-C' (Join-Path $script:repoRoot 'Sense') 'build' '-o' $senseBinary './cmd/sense-api'

    $stage = 'starting isolated PostgreSQL'
    New-Item -ItemType Directory -Path $pgData | Out-Null
    Invoke-Checked $script:initdb '-D' $pgData '-U' 'postgres' '-A' 'trust' '--encoding=UTF8' '--no-locale' '--no-sync'
    $serverOptions = "-h 127.0.0.1 -p $postgresPort -c listen_addresses=127.0.0.1"
    # Do not pipe pg_ctl start output: postgres can inherit the pipeline handle and
    # keep PowerShell waiting until the server exits.
    & $script:pgCtl '-D' $pgData '-l' $pgLog '-o' $serverOptions '-w' 'start'
    if ($LASTEXITCODE -ne 0) {
        throw "required command failed with exit code $LASTEXITCODE"
    }
    $postgresStarted = $true
    Invoke-Checked $script:psql '-X' '-v' 'ON_ERROR_STOP=1' '-d' $adminRootDSN '-f' (Join-Path $script:repoRoot 'deploy\postgres\001_roles.sql')
    Invoke-Checked $script:createdb '-h' '127.0.0.1' '-p' ([string]$postgresPort) '-U' 'postgres' $databaseName
    foreach ($pass in 1..2) {
        foreach ($name in $migrationNames) {
            Invoke-Checked $script:psql '-X' '-v' 'ON_ERROR_STOP=1' '-d' $script:adminDatabaseDSN '-f' (Join-Path $script:repoRoot "deploy\postgres\$name")
        }
    }
    Invoke-Checked $script:psql '-X' '-v' 'ON_ERROR_STOP=1' '-d' $adminRootDSN '-c' 'CREATE ROLE yovision_t014_sense LOGIN IN ROLE sense_app'
    Invoke-Checked $script:psql '-X' '-v' 'ON_ERROR_STOP=1' '-d' $script:adminDatabaseDSN '-c' "INSERT INTO bell.sites(tenant_id,id,name,max_video_channels) VALUES ('tenant-t014','site-t014','T-014 Lab',16); INSERT INTO bell.areas(tenant_id,site_id,id,name,capture_policy) VALUES ('tenant-t014','site-t014','area-t014','T-014 Area','video_allowed');"

    $stage = 'starting independent publishers'
    $sourceMedia = Start-ManagedProcess 'mediamtx-source' $mediaMTX @($sourceConfig)
    $managed.Add($sourceMedia)
    Wait-Port $script:sourceAPIPort
    foreach ($index in 1..$sourceCount) {
        $script:publishers[$index] = Start-Publisher $index
        $managed.Add($script:publishers[$index])
    }
    $publisherReadySeconds = Wait-SourcePublishers
    $productionMedia = Start-ManagedProcess 'mediamtx-production' $mediaMTX @($productionConfig)
    $managed.Add($productionMedia)
    Wait-Port $script:productionAPIPort

    $stage = 'starting Sense Control API'
    $script:controlToken = New-SecretToken
    $tokenDigest = Get-SHA256Hex $script:controlToken
    $cursorKey = ConvertTo-Base64Url (New-RandomBytes 32)
    $localCredential = New-SecretToken
    [IO.File]::WriteAllText($authPath, (([ordered]@{
        version = 1
        principals = @([ordered]@{
            token_sha256 = $tokenDigest
            subject_id = 't014-runner'
            actor_type = 'service'
            tenant_id = 'tenant-t014'
            site_ids = @('site-t014')
            permissions = @('sense.devices.read', 'sense.devices.write')
        })
    } | ConvertTo-Json -Depth 6) + [Environment]::NewLine), [Text.UTF8Encoding]::new($false))
    [IO.File]::WriteAllText($cursorPath, $cursorKey + [Environment]::NewLine, [Text.UTF8Encoding]::new($false))
    $senseEnvironment = @{
        SENSE_HTTP_ADDR = "127.0.0.1:$script:senseHTTPPort"
        SENSE_DB_DRIVER = 'postgres'
        SENSE_DB_DSN = $senseDSN
        SENSE_MEDIAMTX_URL = "http://127.0.0.1:$script:productionAPIPort"
        SENSE_RECONCILE_INTERVAL = '1s'
        SENSE_RECONCILE_LEASE_DURATION = '30s'
        SENSE_RECONCILE_OPERATION_TIMEOUT = '20s'
        SENSE_PROBE_INTERVAL = '1s'
        SENSE_INSTANCE_ID = 't014-lab'
        SENSE_METRICS_ENABLED = 'true'
        SENSE_ORPHAN_SCAN_ENABLED = 'false'
        SENSE_ONVIF_MODE = 'disabled'
        SENSE_CONTROL_API_ENABLED = 'true'
        SENSE_CONTROL_AUTH_FILE = $authPath
        SENSE_CONTROL_CURSOR_KEY_FILE = $cursorPath
        SENSE_CREDENTIAL_SYNTHETIC_ONVIF_USERNAME = 't014-local'
        SENSE_CREDENTIAL_SYNTHETIC_ONVIF_PASSWORD = $localCredential
        SENSE_CREDENTIAL_SYNTHETIC_RTSP_USERNAME = 't014-local'
        SENSE_CREDENTIAL_SYNTHETIC_RTSP_PASSWORD = $localCredential
    }
    $sense = Start-ManagedProcess 'sense-api' $senseBinary @() $senseEnvironment
    $managed.Add($sense)
    Wait-SenseHealth $sense

    $stage = 'creating 16 disabled devices'
    $deviceIDs = [Collections.Generic.List[string]]::new()
    $createEntries = [Collections.Generic.List[object]]::new()
    foreach ($index in 1..$sourceCount) {
        $name = 'synthetic-{0:D2}' -f $index
        $response = Invoke-ControlRequest 'POST' '/api/v1/sites/site-t014/devices' @(201) ([ordered]@{
            serial_number = "t014-$name"
            name = "T-014 source $index"
            modality = 'video'
            capabilities = @('video_capture')
            area_id = 'area-t014'
            endpoint_ref = "rtsp://127.0.0.1:$script:sourceRTSPPort/$name"
            credential_ref = 'env://synthetic'
            desired_state = 'disabled'
        }) @{ 'Idempotency-Key' = "t014-create-$index-$script:runNonce" }
        $created = Convert-ResponseJSON $response
        $deviceIDs.Add([string]$created.id)
        $createEntries.Add([pscustomobject]@{ DeviceID = [string]$created.id; ETag = Get-ResponseETag $response })
    }
    $stage = 'batch enabling 16 devices'
    $firstEnableBatch = Invoke-BatchDesiredState 'enabled' @($createEntries) 'enable-1'
    $firstEnableSeconds = Wait-Convergence 'enabled'
    $firstEnabledInventory = Get-MediaInventory $script:productionAPIPort

    $stage = 'rejecting seventeenth enabled device'
    $overflowResponse = Invoke-ControlRequest 'POST' '/api/v1/sites/site-t014/devices' @(409) ([ordered]@{
        serial_number = 't014-overflow-17'
        name = 'T-014 overflow source'
        modality = 'video'
        capabilities = @('video_capture')
        area_id = 'area-t014'
        endpoint_ref = "rtsp://127.0.0.1:$script:sourceRTSPPort/synthetic-01"
        credential_ref = 'env://synthetic'
        desired_state = 'enabled'
    }) @{ 'Idempotency-Key' = "t014-overflow-17-$script:runNonce" }
    $overflowProblem = Convert-ResponseJSON $overflowResponse
    $overflowCodeProperty = $overflowProblem.PSObject.Properties['code']
    if ($null -eq $overflowCodeProperty) {
        $propertyNames = @($overflowProblem.PSObject.Properties.Name) -join ','
        throw "quota response did not contain a stable code field; properties=$propertyNames"
    }
    $overflowCode = [string]$overflowCodeProperty.Value
    $pageAfterOverflow = Get-DevicePage
    if ($overflowCode -ne 'quota_exceeded' -or @($pageAfterOverflow.items).Count -ne $sourceCount) {
        throw 'the seventeenth enabled device did not fail closed at the site quota'
    }

    $stage = 'batch disabling 16 devices'
    $disableEntries = Get-DeviceEntries @($deviceIDs)
    $disableBatch = Invoke-BatchDesiredState 'disabled' $disableEntries 'disable'
    $disableSeconds = Wait-Convergence 'disabled'
    $disabledInventory = Get-MediaInventory $script:productionAPIPort

    $stage = 'batch re-enabling 16 devices'
    $secondEnableEntries = Get-DeviceEntries @($deviceIDs)
    $secondEnableBatch = Invoke-BatchDesiredState 'enabled' $secondEnableEntries 'enable-2'
    $secondEnableSeconds = Wait-Convergence 'enabled'
    $secondEnabledInventory = Get-MediaInventory $script:productionAPIPort

    $stage = 'injecting four-source fault'
    foreach ($index in $faultIndexes) {
        Stop-ManagedProcess $script:publishers[$index]
    }
    $fault = Wait-PartialFault
    foreach ($index in $faultIndexes) {
        $script:publishers[$index] = Start-Publisher $index
        $managed.Add($script:publishers[$index])
    }
    $faultRecoverySeconds = Wait-Convergence 'enabled'
    $fault.recovery_seconds = $faultRecoverySeconds
    $fault.final_unconverged = 0

    $stage = 'observing stable 16-stream capacity'
    $observation = [Diagnostics.Stopwatch]::StartNew()
    $targetSeconds = $ObservationMinutes * 60
    $senseCPU = [Collections.Generic.List[double]]::new()
    $senseWorking = [Collections.Generic.List[double]]::new()
    $sensePrivate = [Collections.Generic.List[double]]::new()
    $senseHandles = [Collections.Generic.List[double]]::new()
    $mediaCPU = [Collections.Generic.List[double]]::new()
    $mediaWorking = [Collections.Generic.List[double]]::new()
    $mediaPrivate = [Collections.Generic.List[double]]::new()
    $mediaHandles = [Collections.Generic.List[double]]::new()
    $databaseConnections = [Collections.Generic.List[double]]::new()
    $previousSenseCPU = $sense.Process.TotalProcessorTime.TotalSeconds
    $previousMediaCPU = $productionMedia.Process.TotalProcessorTime.TotalSeconds
    $observationSenseCPUStart = $previousSenseCPU
    $observationMediaCPUStart = $previousMediaCPU
    $previousSampleAt = 0.0
    $initialInventory = Get-MediaInventory $script:productionAPIPort
    $samples = 0
    $requiredSamples = [int][Math]::Ceiling($targetSeconds / $sampleSeconds)
    $maximumUnconverged = 0
    while ($samples -lt $requiredSamples) {
        $nextSampleAt = ($samples + 1) * $sampleSeconds
        $delayMilliseconds = [int][Math]::Ceiling(($nextSampleAt - $observation.Elapsed.TotalSeconds) * 1000)
        if ($delayMilliseconds -gt 0) {
            Start-Sleep -Milliseconds $delayMilliseconds
        }
        Assert-Alive $sourceMedia
        Assert-Alive $productionMedia
        Assert-Alive $sense
        foreach ($publisher in $script:publishers.Values) { Assert-Alive $publisher }
        $elapsed = $observation.Elapsed.TotalSeconds
        $delta = $elapsed - $previousSampleAt
        $senseSample = Get-ProcessResourceSample $sense $previousSenseCPU $delta
        $mediaSample = Get-ProcessResourceSample $productionMedia $previousMediaCPU $delta
        $previousSenseCPU = $senseSample.TotalCPU
        $previousMediaCPU = $mediaSample.TotalCPU
        $previousSampleAt = $elapsed
        $senseCPU.Add($senseSample.CPUPercent)
        $senseWorking.Add($senseSample.WorkingSetMiB)
        $sensePrivate.Add($senseSample.PrivateMiB)
        $senseHandles.Add($senseSample.Handles)
        $mediaCPU.Add($mediaSample.CPUPercent)
        $mediaWorking.Add($mediaSample.WorkingSetMiB)
        $mediaPrivate.Add($mediaSample.PrivateMiB)
        $mediaHandles.Add($mediaSample.Handles)
        $databaseConnections.Add([double](Get-PostgresScalar "SELECT count(*) FROM pg_stat_activity WHERE datname = '$databaseName'"))
        $page = Get-DevicePage
        $inventory = Get-MediaInventory $script:productionAPIPort
        $unconverged = [int](Get-SenseMetric 'sense_reconcile_devices' '{state="unconverged"}')
        $maximumUnconverged = [Math]::Max($maximumUnconverged, $unconverged)
        if (@($page.items).Count -ne $sourceCount -or
            @($page.items | Where-Object { $_.desired_state -ne 'enabled' -or $_.actual_state -ne 'online' -or -not $_.converged }).Count -ne 0 -or
            $inventory.Configured -ne $sourceCount -or $inventory.Runtime -ne $sourceCount -or
            $inventory.Online -ne $sourceCount -or $unconverged -ne 0) {
            throw 'stability observation detected a non-converged stream'
        }
        $samples++
    }
    $observationSeconds = [Math]::Round($observation.Elapsed.TotalSeconds, 1)
    $finalInventory = Get-MediaInventory $script:productionAPIPort
    $sense.Process.Refresh()
    $productionMedia.Process.Refresh()
    $senseCPUTotal = $sense.Process.TotalProcessorTime.TotalSeconds - $observationSenseCPUStart
    $mediaCPUTotal = $productionMedia.Process.TotalProcessorTime.TotalSeconds - $observationMediaCPUStart
    $inboundDelta = [UInt64]($finalInventory.InboundBytes - $initialInventory.InboundBytes)
    $frameErrorDelta = [UInt64]($finalInventory.FrameErrors - $initialInventory.FrameErrors)
    $inboundMbps = if ($observationSeconds -gt 0) { ($inboundDelta * 8.0) / $observationSeconds / 1000000.0 } else { 0.0 }
    $databaseSizeBytes = [Int64](Get-PostgresScalar "SELECT pg_database_size('$databaseName')")
    $finalUnconverged = [int](Get-SenseMetric 'sense_reconcile_devices' '{state="unconverged"}')
    if ($observationSeconds -lt $targetSeconds -or $finalUnconverged -ne 0 -or
        $maximumUnconverged -ne 0 -or $samples -lt $requiredSamples) {
        throw 'formal observation did not satisfy sampling and convergence requirements'
    }

    $memoryGiB = 0.0
    try { $memoryGiB = (Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory / 1GB } catch {}
    $gitCommit = (& git -C $script:repoRoot rev-parse HEAD | Out-String).Trim()
    $success = $true
    $resultJSON = [ordered]@{
        success = $true
        formal_eligible = $ObservationMinutes -ge 30
        scope = 'laboratory_software_baseline'
        source_count = $sourceCount
        independent_publishers = $sourceCount
        fixture = [ordered]@{ width = 640; height = 360; fps = 10; codec = 'H.264'; audio = $false; publish_mode = 'preencoded_copy' }
        versions = [ordered]@{
            repository_commit = $gitCommit
            sense_sha256 = (Get-FileHash -LiteralPath $senseBinary -Algorithm SHA256).Hash.ToLowerInvariant()
            mediamtx = $mediaMTXVersion
            mediamtx_package_sha256 = $mediaMTXSHA256
            mediamtx_executable_sha256 = $mediaMTXExecutableSHA256
            postgres = $postgresVersion
            ffmpeg = [string]$ffmpegVersion
            go = $goVersion
        }
        host = [ordered]@{ logical_processors = [Environment]::ProcessorCount; total_memory_gib = [Math]::Round($memoryGiB, 2) }
        quota = [ordered]@{ configured = 16; accepted_devices = 16; seventeenth_rejected = $true; error_code = 'quota_exceeded' }
        publishers_ready_seconds = $publisherReadySeconds
        batch = [ordered]@{
            enable_first = $firstEnableBatch
            disable = $disableBatch
            enable_second = $secondEnableBatch
        }
        convergence_seconds = [ordered]@{
            enable_first = $firstEnableSeconds
            disable = $disableSeconds
            enable_second = $secondEnableSeconds
        }
        configured_paths = [ordered]@{
            after_enable_first = $firstEnabledInventory.Configured
            after_disable = $disabledInventory.Configured
            after_enable_second = $secondEnabledInventory.Configured
        }
        fault_recovery = $fault
        observation = [ordered]@{
            requested_minutes = $ObservationMinutes
            seconds = $observationSeconds
            sample_interval_seconds = $sampleSeconds
            samples = $samples
            maximum_unconverged = $maximumUnconverged
            final_unconverged = $finalUnconverged
            final_online_paths = $finalInventory.Online
            aggregate_inbound_mbps = [Math]::Round($inboundMbps, 3)
            inbound_frame_errors = $frameErrorDelta
        }
        resources = [ordered]@{
            sense = [ordered]@{
                cpu_average_percent = [Math]::Round((Get-Average $senseCPU), 3)
                cpu_peak_percent = [Math]::Round((Get-Maximum $senseCPU), 3)
                cpu_total_seconds = [Math]::Round($senseCPUTotal, 3)
                working_set_peak_mib = [Math]::Round((Get-Maximum $senseWorking), 2)
                private_peak_mib = [Math]::Round((Get-Maximum $sensePrivate), 2)
                handles_peak = [int](Get-Maximum $senseHandles)
            }
            mediamtx = [ordered]@{
                cpu_average_percent = [Math]::Round((Get-Average $mediaCPU), 3)
                cpu_peak_percent = [Math]::Round((Get-Maximum $mediaCPU), 3)
                cpu_total_seconds = [Math]::Round($mediaCPUTotal, 3)
                working_set_peak_mib = [Math]::Round((Get-Maximum $mediaWorking), 2)
                private_peak_mib = [Math]::Round((Get-Maximum $mediaPrivate), 2)
                handles_peak = [int](Get-Maximum $mediaHandles)
            }
            postgres_peak_connections = [int](Get-Maximum $databaseConnections)
            postgres_database_mib = [Math]::Round($databaseSizeBytes / 1MB, 2)
        }
        limitations = @('synthetic_sources', 'no_customer_network', 'no_recording', 'no_downstream_readers', 'no_ai_or_gpu', 'not_a_production_sla')
    } | ConvertTo-Json -Depth 12
}
catch {
    $failure = "T-014 stage '$stage' failed: $($_.Exception.Message)"
}
finally {
    Stop-AllManagedProcesses @($managed)
    if ($postgresStarted) {
        & $script:pgCtl '-D' $pgData '-m' 'fast' '-w' 'stop' | Out-Null
        $postgresStopped = $LASTEXITCODE -eq 0
    }
    if (-not $postgresStarted -or $postgresStopped) {
        $resolvedSession = [IO.Path]::GetFullPath($script:session)
        $expectedPrefix = $runtimeResolved.TrimEnd('\') + '\'
        if (-not $resolvedSession.StartsWith($expectedPrefix, [StringComparison]::OrdinalIgnoreCase) -or
            [IO.Path]::GetFileName($resolvedSession) -notlike 'session-*') {
            throw 'refusing to clean an unexpected T-014 session path'
        }
        if (Test-Path -LiteralPath $resolvedSession) {
            Remove-SessionDirectory $resolvedSession
        }
    }
    else {
        Write-Warning 'Temporary PostgreSQL did not stop; the session directory was retained for manual recovery'
    }
    $after5432 = @(
        Get-NetTCPConnection -State Listen -LocalPort 5432 -ErrorAction SilentlyContinue |
            Select-Object -ExpandProperty OwningProcess -Unique | Sort-Object
    )
    if (($existing5432 -join ',') -ne ($after5432 -join ',')) {
        throw 'the existing PostgreSQL listener on port 5432 changed during T-014'
    }
}
if (-not $success) {
    throw $failure
}
Write-Result $resultJSON