[CmdletBinding()] param( [string]$PgRoot = 'D:\pgsql17', [string]$RuntimeRoot = (Join-Path ([IO.Path]::GetTempPath()) 'yovision-t014'), [ValidateRange(1, 1440)] [int]$ObservationMinutes = 30, [string]$OutputPath = '', [switch]$PreflightOnly ) $ErrorActionPreference = 'Stop' $ProgressPreference = 'SilentlyContinue' Set-StrictMode -Version Latest $sourceCount = 16 $faultIndexes = @(5, 6, 7, 8) $sampleSeconds = 10 $mediaMTXVersion = 'v1.19.3' $mediaMTXSHA256 = '5d82148d1032a6a190d9909a2997d9989457aaadf49af87dd02cd4512d31bebe' $mediaMTXExecutableSHA256 = '1cda85249312cb9463f9f94c5a712b9f160c9af3fd9490f0d4723911d7880e05' $migrationNames = @( '002_bell.sql', '003_sense.sql', '004_privileges.sql', '005_area_policy.sql', '006_device_operation_outbox.sql', '007_privileges_area_audit.sql', '008_control_api.sql', '009_privileges_control_api.sql', '010_reconcile_safety.sql', '011_privileges_reconcile_safety.sql' ) function ConvertTo-Base64Url([byte[]]$Bytes) { return [Convert]::ToBase64String($Bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_') } function New-RandomBytes([int]$Count) { $bytes = [byte[]]::new($Count) [Security.Cryptography.RandomNumberGenerator]::Fill($bytes) return $bytes } function New-SecretToken { return ConvertTo-Base64Url (New-RandomBytes 32) } function Get-SHA256Hex([string]$Value) { $bytes = [Text.Encoding]::UTF8.GetBytes($Value) $digest = [Security.Cryptography.SHA256]::HashData($bytes) return ([Convert]::ToHexString($digest)).ToLowerInvariant() } function Get-FreeTcpPort([Collections.Generic.HashSet[int]]$Reserved) { while ($true) { $listener = [Net.Sockets.TcpListener]::new([Net.IPAddress]::Loopback, 0) $listener.Start() $port = ([Net.IPEndPoint]$listener.LocalEndpoint).Port $listener.Stop() if ($Reserved.Add($port)) { return $port } } } function Wait-Port([int]$Port, [int]$TimeoutSeconds = 30) { $watch = [Diagnostics.Stopwatch]::StartNew() while ($watch.Elapsed.TotalSeconds -lt $TimeoutSeconds) { $client = [Net.Sockets.TcpClient]::new() try { $task = $client.ConnectAsync('127.0.0.1', $Port) if ($task.Wait(500) -and $client.Connected) { return } } catch { } finally { $client.Dispose() } Start-Sleep -Milliseconds 250 } throw "local process did not listen before timeout" } function Start-ManagedProcess( [string]$Name, [string]$FilePath, [string[]]$Arguments, [hashtable]$Environment = @{} ) { $start = [Diagnostics.ProcessStartInfo]::new() $start.FileName = $FilePath $start.WorkingDirectory = $script:session $start.UseShellExecute = $false $start.CreateNoWindow = $true $start.RedirectStandardOutput = $true $start.RedirectStandardError = $true foreach ($argument in $Arguments) { $start.ArgumentList.Add($argument) } foreach ($entry in $Environment.GetEnumerator()) { $start.Environment[$entry.Key] = [string]$entry.Value } $process = [Diagnostics.Process]::new() $process.StartInfo = $start if (-not $process.Start()) { throw "failed to start required process: $Name" } return [pscustomobject]@{ Name = $Name Process = $process Stdout = $process.StandardOutput.ReadToEndAsync() Stderr = $process.StandardError.ReadToEndAsync() } } function Stop-ManagedProcess($Managed) { if ($null -eq $Managed -or $null -eq $Managed.Process) { return } if (-not $Managed.Process.HasExited) { $Managed.Process.Kill() $Managed.Process.WaitForExit(10000) | Out-Null } } function Stop-AllManagedProcesses($Items) { $processes = @($Items | Where-Object { $null -ne $_ -and $null -ne $_.Process }) foreach ($item in $processes) { try { if (-not $item.Process.HasExited) { $item.Process.Kill() } } catch { if (-not $item.Process.HasExited) { throw } } } $cleanupWatch = [Diagnostics.Stopwatch]::StartNew() while ($cleanupWatch.Elapsed.TotalSeconds -lt 10 -and @($processes | Where-Object { -not $_.Process.HasExited }).Count -gt 0) { Start-Sleep -Milliseconds 100 } $remaining = @($processes | Where-Object { -not $_.Process.HasExited }) if ($remaining.Count -gt 0) { Write-Warning "managed processes did not exit within the shared cleanup deadline: $($remaining.Count)" } foreach ($item in $processes) { if ($item.Process.HasExited) { $item.Stdout.Wait(2000) | Out-Null $item.Stderr.Wait(2000) | Out-Null $item.Process.Dispose() } } } function Remove-SessionDirectory([string]$Path) { Get-ChildItem -LiteralPath $Path -Recurse -Force -ErrorAction SilentlyContinue | ForEach-Object { $_.Attributes = 'Normal' } (Get-Item -LiteralPath $Path -Force).Attributes = 'Directory' $deleteWatch = [Diagnostics.Stopwatch]::StartNew() while (Test-Path -LiteralPath $Path) { try { Remove-Item -LiteralPath $Path -Recurse -Force -ErrorAction Stop } catch { if ($deleteWatch.Elapsed.TotalSeconds -ge 10) { throw } Start-Sleep -Milliseconds 200 } } } function Assert-Alive($Managed) { if ($null -eq $Managed -or $Managed.Process.HasExited) { $code = if ($null -eq $Managed) { 'not-started' } else { $Managed.Process.ExitCode } throw "required process exited: $($Managed.Name), code=$code" } } function Invoke-Checked { param( [Parameter(Mandatory)] [string]$FilePath, [Parameter(ValueFromRemainingArguments)] [string[]]$Arguments ) & $FilePath @Arguments 2>&1 | Out-Null if ($LASTEXITCODE -ne 0) { throw "required command failed with exit code $LASTEXITCODE" } } function Invoke-ControlRequest( [string]$Method, [string]$Path, [int[]]$ExpectedStatus, $Body = $null, [hashtable]$ExtraHeaders = @{} ) { $headers = @{ Authorization = "Bearer $script:controlToken" Accept = 'application/json' } foreach ($entry in $ExtraHeaders.GetEnumerator()) { $headers[$entry.Key] = $entry.Value } $parameters = @{ Method = $Method Uri = "http://127.0.0.1:$script:senseHTTPPort$Path" Headers = $headers TimeoutSec = 15 NoProxy = $true SkipHttpErrorCheck = $true } if ($null -ne $Body) { $parameters.ContentType = 'application/json' $parameters.Body = ($Body | ConvertTo-Json -Depth 10 -Compress) } $response = Invoke-WebRequest @parameters $statusCode = [int]$response.StatusCode $statusMatches = $false foreach ($expected in $ExpectedStatus) { if ([int]$expected -eq $statusCode) { $statusMatches = $true break } } if (-not $statusMatches) { $code = 'unknown' try { $code = ((Get-ResponseText $response) | ConvertFrom-Json).code } catch { } throw "Control API returned unexpected status $statusCode, expected=$($ExpectedStatus -join ','), code=$code" } return $response } function Get-ResponseText($Response) { if ($Response.Content -is [byte[]]) { return [Text.Encoding]::UTF8.GetString($Response.Content) } return [string]$Response.Content } function Convert-ResponseJSON($Response) { return ((Get-ResponseText $Response) | ConvertFrom-Json) } function Get-ResponseETag($Response) { $value = [string]$Response.Headers.ETag if ([string]::IsNullOrWhiteSpace($value)) { throw 'Control API response did not contain ETag' } return $value } function Get-DevicePage { $response = Invoke-ControlRequest 'GET' '/api/v1/sites/site-t014/devices?limit=100' @(200) return Convert-ResponseJSON $response } function Get-DeviceEntries([string[]]$DeviceIDs) { $entries = [Collections.Generic.List[object]]::new() foreach ($id in $DeviceIDs) { $response = Invoke-ControlRequest 'GET' "/api/v1/sites/site-t014/devices/$id" @(200) $entries.Add([pscustomobject]@{ DeviceID = $id ETag = Get-ResponseETag $response }) } return @($entries) } function Invoke-BatchDesiredState( [string]$DesiredState, [object[]]$Entries, [string]$KeySuffix ) { $items = @($Entries | ForEach-Object { [ordered]@{ device_id = $_.DeviceID etag = $_.ETag desired_state = $DesiredState } }) $watch = [Diagnostics.Stopwatch]::StartNew() $response = Invoke-ControlRequest 'POST' '/api/v1/sites/site-t014/devices:batchDesiredState' @(202) ([ordered]@{ items = $items reason = "T-014 laboratory $DesiredState baseline" }) @{ 'Idempotency-Key' = "t014-batch-$KeySuffix-$script:runNonce" } $operation = Convert-ResponseJSON $response $results = @($operation.results) if ($operation.status -ne 'succeeded' -or $results.Count -ne $sourceCount -or @($results | Where-Object { $_.status -ne 'succeeded' }).Count -ne 0) { throw "batch desired-state operation was not fully successful" } return [ordered]@{ status = [string]$operation.status item_count = $results.Count request_seconds = [Math]::Round($watch.Elapsed.TotalSeconds, 3) } } function Get-MediaInventory([int]$APIPort) { $config = Invoke-RestMethod -Method Get -Uri "http://127.0.0.1:$APIPort/v3/config/paths/list?page=0&itemsPerPage=100" -TimeoutSec 5 -NoProxy $runtime = Invoke-RestMethod -Method Get -Uri "http://127.0.0.1:$APIPort/v3/paths/list?page=0&itemsPerPage=100" -TimeoutSec 5 -NoProxy $items = @($runtime.items) [UInt64]$inboundBytes = 0 [UInt64]$frameErrors = 0 foreach ($item in $items) { if ($null -ne $item.inboundBytes) { $inboundBytes += [UInt64]$item.inboundBytes } if ($null -ne $item.inboundFramesInError) { $frameErrors += [UInt64]$item.inboundFramesInError } } return [pscustomobject]@{ Configured = [int]$config.itemCount Runtime = [int]$runtime.itemCount Online = @($items | Where-Object { $_.online -eq $true -and $_.available -eq $true }).Count InboundBytes = $inboundBytes FrameErrors = $frameErrors } } function Get-SenseMetric([string]$Name, [string]$Labels) { $response = Invoke-WebRequest -Method Get -Uri "http://127.0.0.1:$script:senseHTTPPort/metrics" -TimeoutSec 5 -NoProxy $prefix = [Regex]::Escape($Name + $Labels) $match = [Regex]::Match((Get-ResponseText $response), "(?m)^$prefix\s+([-+0-9.eE]+)$") if (-not $match.Success) { throw "required Sense metric was not found: $Name" } return [double]::Parse($match.Groups[1].Value, [Globalization.CultureInfo]::InvariantCulture) } function Wait-SenseHealth($Managed, [int]$TimeoutSeconds = 45) { $watch = [Diagnostics.Stopwatch]::StartNew() while ($watch.Elapsed.TotalSeconds -lt $TimeoutSeconds) { Assert-Alive $Managed try { $ready = Invoke-RestMethod -Method Get -Uri "http://127.0.0.1:$script:senseHTTPPort/readyz" -TimeoutSec 2 -NoProxy if ($ready.status -eq 'ready') { return } } catch { } Start-Sleep -Milliseconds 500 } throw 'Sense readiness endpoint did not become ready' } function Wait-SourcePublishers([int]$TimeoutSeconds = 90) { $watch = [Diagnostics.Stopwatch]::StartNew() while ($watch.Elapsed.TotalSeconds -lt $TimeoutSeconds) { foreach ($publisher in $script:publishers.Values) { Assert-Alive $publisher } try { $inventory = Get-MediaInventory $script:sourceAPIPort if ($inventory.Runtime -eq $sourceCount -and $inventory.Online -eq $sourceCount) { return [Math]::Round($watch.Elapsed.TotalSeconds, 1) } } catch { } Start-Sleep -Seconds 1 } throw 'independent source publishers did not all become online' } function Wait-Convergence([ValidateSet('enabled', 'disabled')] [string]$DesiredState, [int]$TimeoutSeconds = 300) { $watch = [Diagnostics.Stopwatch]::StartNew() $last = 'no observation' while ($watch.Elapsed.TotalSeconds -lt $TimeoutSeconds) { try { $page = Get-DevicePage $devices = @($page.items) $inventory = Get-MediaInventory $script:productionAPIPort $totalMetric = Get-SenseMetric 'sense_reconcile_devices' '{state="total"}' $unconvergedMetric = Get-SenseMetric 'sense_reconcile_devices' '{state="unconverged"}' if ($DesiredState -eq 'enabled') { $stateOK = $devices.Count -eq $sourceCount -and @($devices | Where-Object { $_.desired_state -ne 'enabled' -or $_.actual_state -ne 'online' -or -not $_.converged }).Count -eq 0 $mediaOK = $inventory.Configured -eq $sourceCount -and $inventory.Runtime -eq $sourceCount -and $inventory.Online -eq $sourceCount $metricsOK = $totalMetric -eq $sourceCount -and $unconvergedMetric -eq 0 } else { $stateOK = $devices.Count -eq $sourceCount -and @($devices | Where-Object { $_.desired_state -ne 'disabled' -or $_.actual_state -ne 'offline' -or -not $_.converged }).Count -eq 0 $mediaOK = $inventory.Configured -eq 0 -and $inventory.Runtime -eq 0 $metricsOK = $totalMetric -eq 0 -and $unconvergedMetric -eq 0 } $last = "devices=$($devices.Count), configured=$($inventory.Configured), runtime=$($inventory.Runtime), online=$($inventory.Online), unconverged=$unconvergedMetric" if ($stateOK -and $mediaOK -and $metricsOK) { return [Math]::Round($watch.Elapsed.TotalSeconds, 1) } } catch { $last = $_.Exception.Message } Start-Sleep -Seconds 1 } throw "devices did not reach $DesiredState convergence: $last" } function Wait-PartialFault([int]$TimeoutSeconds = 120) { $watch = [Diagnostics.Stopwatch]::StartNew() while ($watch.Elapsed.TotalSeconds -lt $TimeoutSeconds) { try { $page = Get-DevicePage $devices = @($page.items) $online = @($devices | Where-Object { $_.actual_state -eq 'online' }).Count $notOnline = $devices.Count - $online if ($devices.Count -eq $sourceCount -and $online -eq ($sourceCount - $faultIndexes.Count) -and $notOnline -eq $faultIndexes.Count) { return [ordered]@{ detection_seconds = [Math]::Round($watch.Elapsed.TotalSeconds, 1) affected = $notOnline unaffected_online = $online } } } catch { } Start-Sleep -Seconds 1 } throw 'four-source fault was not isolated to exactly four devices' } function Start-Publisher([int]$Index) { $name = 'synthetic-{0:D2}' -f $Index return Start-ManagedProcess "publisher-$Index" $script:ffmpeg @( '-hide_banner', '-loglevel', 'warning', '-re', '-stream_loop', '-1', '-i', $script:fixturePath, '-map', '0:v:0', '-c', 'copy', '-an', '-f', 'rtsp', '-rtsp_transport', 'tcp', "rtsp://127.0.0.1:$script:sourceRTSPPort/$name" ) } function Get-ProcessResourceSample($Managed, [double]$PreviousCPU, [double]$ElapsedSeconds) { Assert-Alive $Managed $Managed.Process.Refresh() $totalCPU = $Managed.Process.TotalProcessorTime.TotalSeconds $normalizedCPU = 0.0 if ($ElapsedSeconds -gt 0) { $normalizedCPU = (($totalCPU - $PreviousCPU) / $ElapsedSeconds / [Environment]::ProcessorCount) * 100 } return [pscustomobject]@{ TotalCPU = $totalCPU CPUPercent = [Math]::Max(0, $normalizedCPU) WorkingSetMiB = $Managed.Process.WorkingSet64 / 1MB PrivateMiB = $Managed.Process.PrivateMemorySize64 / 1MB Handles = $Managed.Process.HandleCount } } function Get-Average([Collections.Generic.List[double]]$Values) { if ($Values.Count -eq 0) { return 0.0 } return ($Values | Measure-Object -Average).Average } function Get-Maximum([Collections.Generic.List[double]]$Values) { if ($Values.Count -eq 0) { return 0.0 } return ($Values | Measure-Object -Maximum).Maximum } function Get-PostgresScalar([string]$Query) { $value = & $script:psql -X -A -t -v ON_ERROR_STOP=1 -d $script:adminDatabaseDSN -c $Query if ($LASTEXITCODE -ne 0) { throw 'PostgreSQL observation query failed' } return (($value | Out-String).Trim()) } function Write-Result([string]$JSON) { if (-not [string]::IsNullOrWhiteSpace($OutputPath)) { $resolvedOutput = [IO.Path]::GetFullPath($OutputPath) $repoPrefix = $script:repoRoot.TrimEnd('\') + '\' if ($resolvedOutput.StartsWith($repoPrefix, [StringComparison]::OrdinalIgnoreCase)) { throw 'capacity result output must stay outside the repository' } $parent = Split-Path -Parent $resolvedOutput if (-not (Test-Path -LiteralPath $parent -PathType Container)) { throw 'capacity result output directory does not exist' } [IO.File]::WriteAllText($resolvedOutput, $JSON + [Environment]::NewLine, [Text.UTF8Encoding]::new($false)) } Write-Output $JSON } $script:repoRoot = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot '..\..')).Path $pgRootResolved = (Resolve-Path -LiteralPath $PgRoot).Path $pgBin = Join-Path $pgRootResolved 'bin' $script:initdb = Join-Path $pgBin 'initdb.exe' $script:pgCtl = Join-Path $pgBin 'pg_ctl.exe' $script:psql = Join-Path $pgBin 'psql.exe' $script:createdb = Join-Path $pgBin 'createdb.exe' $postgres = Join-Path $pgBin 'postgres.exe' foreach ($required in @($script:initdb, $script:pgCtl, $script:psql, $script:createdb, $postgres)) { if (-not (Test-Path -LiteralPath $required -PathType Leaf)) { throw 'PostgreSQL executable is missing under the selected PgRoot' } } $postgresVersion = (& $postgres --version 2>&1 | Out-String).Trim() if ($LASTEXITCODE -ne 0 -or $postgresVersion -notmatch 'PostgreSQL\) 17\.10$') { throw 'T-014 requires the frozen PostgreSQL 17.10 binaries' } $script:ffmpeg = (Get-Command ffmpeg -ErrorAction Stop).Source $ffmpegVersion = (& $script:ffmpeg -version 2>&1 | Select-Object -First 1) if ($ffmpegVersion -notmatch '^ffmpeg version 8\.1\.2') { throw 'T-014 requires the frozen FFmpeg 8.1.2 build' } $goVersion = (& go version 2>&1 | Out-String).Trim() if ($LASTEXITCODE -ne 0 -or $goVersion -notmatch '^go version go1\.23\.') { throw 'T-014 requires the frozen Go 1.23 toolchain' } $systemTemp = [IO.Path]::GetFullPath([IO.Path]::GetTempPath()).TrimEnd('\') $runtimeResolved = [IO.Path]::GetFullPath($RuntimeRoot).TrimEnd('\') $runtimePrefix = $systemTemp + '\' if (-not $runtimeResolved.StartsWith($runtimePrefix, [StringComparison]::OrdinalIgnoreCase) -or [IO.Path]::GetFileName($runtimeResolved) -notlike 'yovision-t014*') { throw 'RuntimeRoot must be a yovision-t014 directory under the system temporary directory' } $mediaDirectory = Join-Path $runtimeResolved 'mediamtx-v1.19.3' $mediaMTX = Join-Path $mediaDirectory 'mediamtx.exe' if ($PreflightOnly) { $memoryGiB = 0.0 try { $memoryGiB = (Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory / 1GB } catch {} Write-Result (([ordered]@{ success = $true preflight_only = $true source_count = $sourceCount observation_minutes = $ObservationMinutes formal_eligible = $ObservationMinutes -ge 30 mediamtx_cached = Test-Path -LiteralPath $mediaMTX -PathType Leaf postgres_version = $postgresVersion ffmpeg_version = [string]$ffmpegVersion go_version = $goVersion logical_processors = [Environment]::ProcessorCount total_memory_gib = [Math]::Round($memoryGiB, 2) } | ConvertTo-Json -Depth 5)) exit 0 } New-Item -ItemType Directory -Path $runtimeResolved -Force | Out-Null if (-not (Test-Path -LiteralPath $mediaMTX -PathType Leaf)) { $zip = Join-Path $runtimeResolved 'mediamtx_v1.19.3_windows_amd64.zip' Invoke-WebRequest 'https://github.com/bluenviron/mediamtx/releases/download/v1.19.3/mediamtx_v1.19.3_windows_amd64.zip' -OutFile $zip $actualHash = (Get-FileHash -LiteralPath $zip -Algorithm SHA256).Hash.ToLowerInvariant() if ($actualHash -ne $mediaMTXSHA256) { throw 'MediaMTX checksum mismatch' } New-Item -ItemType Directory -Path $mediaDirectory -Force | Out-Null Expand-Archive -LiteralPath $zip -DestinationPath $mediaDirectory -Force } if ((Get-FileHash -LiteralPath $mediaMTX -Algorithm SHA256).Hash.ToLowerInvariant() -ne $mediaMTXExecutableSHA256) { throw 'MediaMTX executable checksum mismatch' } $script:runNonce = [Guid]::NewGuid().ToString('N') $script:session = Join-Path $runtimeResolved ("session-$script:runNonce") New-Item -ItemType Directory -Path $script:session | Out-Null $reservedPorts = [Collections.Generic.HashSet[int]]::new() $script:sourceRTSPPort = Get-FreeTcpPort $reservedPorts $script:sourceAPIPort = Get-FreeTcpPort $reservedPorts $productionRTSPPort = Get-FreeTcpPort $reservedPorts $script:productionAPIPort = Get-FreeTcpPort $reservedPorts $productionMetricsPort = Get-FreeTcpPort $reservedPorts $script:senseHTTPPort = Get-FreeTcpPort $reservedPorts $postgresPort = Get-FreeTcpPort $reservedPorts $sourceConfig = Join-Path $script:session 'mediamtx-source.yml' $productionConfig = Join-Path $script:session 'mediamtx-production.yml' $sourceLines = [Collections.Generic.List[string]]::new() @( 'logLevel: warn', "rtspAddress: 127.0.0.1:$script:sourceRTSPPort", 'rtspTransports: [tcp]', 'api: true', "apiAddress: 127.0.0.1:$script:sourceAPIPort", 'metrics: false', 'rtmp: false', 'hls: false', 'webrtc: false', 'srt: false', 'moq: false', 'paths:' ) | ForEach-Object { $sourceLines.Add($_) } foreach ($index in 1..$sourceCount) { $sourceLines.Add((' synthetic-{0:D2}:' -f $index)) $sourceLines.Add(' source: publisher') } [IO.File]::WriteAllLines($sourceConfig, $sourceLines, [Text.UTF8Encoding]::new($false)) [IO.File]::WriteAllLines($productionConfig, @( 'logLevel: warn', "rtspAddress: 127.0.0.1:$productionRTSPPort", 'rtspTransports: [tcp]', 'api: true', "apiAddress: 127.0.0.1:$script:productionAPIPort", 'metrics: true', "metricsAddress: 127.0.0.1:$productionMetricsPort", 'rtmp: false', 'hls: false', 'webrtc: false', 'srt: false', 'moq: false', 'paths: {}' ), [Text.UTF8Encoding]::new($false)) $script:fixturePath = Join-Path $script:session 'fixture.mp4' $senseBinary = Join-Path $script:session 'sense-api.exe' $authPath = Join-Path $script:session 'control-auth.json' $cursorPath = Join-Path $script:session 'cursor.key' $pgData = Join-Path $script:session 'pgdata' $pgLog = Join-Path $script:session 'postgres.log' $databaseName = 'yovision_t014' $adminRootDSN = "postgres://postgres@127.0.0.1:$postgresPort/postgres?sslmode=disable" $script:adminDatabaseDSN = "postgres://postgres@127.0.0.1:$postgresPort/${databaseName}?sslmode=disable" $senseDSN = "postgres://yovision_t014_sense@127.0.0.1:$postgresPort/${databaseName}?sslmode=disable" $existing5432 = @( Get-NetTCPConnection -State Listen -LocalPort 5432 -ErrorAction SilentlyContinue | Select-Object -ExpandProperty OwningProcess -Unique | Sort-Object ) $managed = [Collections.Generic.List[object]]::new() $script:publishers = @{} $postgresStarted = $false $postgresStopped = $false $success = $false $stage = 'preparing fixtures' $failure = $null $resultJSON = $null try { Invoke-Checked $script:ffmpeg '-hide_banner' '-loglevel' 'error' '-f' 'lavfi' '-i' 'testsrc2=size=640x360:rate=10' '-t' '30' '-c:v' 'libx264' '-preset' 'ultrafast' '-tune' 'zerolatency' '-pix_fmt' 'yuv420p' '-g' '10' '-an' '-movflags' '+faststart' '-y' $script:fixturePath Invoke-Checked 'go' '-C' (Join-Path $script:repoRoot 'Sense') 'build' '-o' $senseBinary './cmd/sense-api' $stage = 'starting isolated PostgreSQL' New-Item -ItemType Directory -Path $pgData | Out-Null Invoke-Checked $script:initdb '-D' $pgData '-U' 'postgres' '-A' 'trust' '--encoding=UTF8' '--no-locale' '--no-sync' $serverOptions = "-h 127.0.0.1 -p $postgresPort -c listen_addresses=127.0.0.1" # Do not pipe pg_ctl start output: postgres can inherit the pipeline handle and # keep PowerShell waiting until the server exits. & $script:pgCtl '-D' $pgData '-l' $pgLog '-o' $serverOptions '-w' 'start' if ($LASTEXITCODE -ne 0) { throw "required command failed with exit code $LASTEXITCODE" } $postgresStarted = $true Invoke-Checked $script:psql '-X' '-v' 'ON_ERROR_STOP=1' '-d' $adminRootDSN '-f' (Join-Path $script:repoRoot 'deploy\postgres\001_roles.sql') Invoke-Checked $script:createdb '-h' '127.0.0.1' '-p' ([string]$postgresPort) '-U' 'postgres' $databaseName foreach ($pass in 1..2) { foreach ($name in $migrationNames) { Invoke-Checked $script:psql '-X' '-v' 'ON_ERROR_STOP=1' '-d' $script:adminDatabaseDSN '-f' (Join-Path $script:repoRoot "deploy\postgres\$name") } } Invoke-Checked $script:psql '-X' '-v' 'ON_ERROR_STOP=1' '-d' $adminRootDSN '-c' 'CREATE ROLE yovision_t014_sense LOGIN IN ROLE sense_app' Invoke-Checked $script:psql '-X' '-v' 'ON_ERROR_STOP=1' '-d' $script:adminDatabaseDSN '-c' "INSERT INTO bell.sites(tenant_id,id,name,max_video_channels) VALUES ('tenant-t014','site-t014','T-014 Lab',16); INSERT INTO bell.areas(tenant_id,site_id,id,name,capture_policy) VALUES ('tenant-t014','site-t014','area-t014','T-014 Area','video_allowed');" $stage = 'starting independent publishers' $sourceMedia = Start-ManagedProcess 'mediamtx-source' $mediaMTX @($sourceConfig) $managed.Add($sourceMedia) Wait-Port $script:sourceAPIPort foreach ($index in 1..$sourceCount) { $script:publishers[$index] = Start-Publisher $index $managed.Add($script:publishers[$index]) } $publisherReadySeconds = Wait-SourcePublishers $productionMedia = Start-ManagedProcess 'mediamtx-production' $mediaMTX @($productionConfig) $managed.Add($productionMedia) Wait-Port $script:productionAPIPort $stage = 'starting Sense Control API' $script:controlToken = New-SecretToken $tokenDigest = Get-SHA256Hex $script:controlToken $cursorKey = ConvertTo-Base64Url (New-RandomBytes 32) $localCredential = New-SecretToken [IO.File]::WriteAllText($authPath, (([ordered]@{ version = 1 principals = @([ordered]@{ token_sha256 = $tokenDigest subject_id = 't014-runner' actor_type = 'service' tenant_id = 'tenant-t014' site_ids = @('site-t014') permissions = @('sense.devices.read', 'sense.devices.write') }) } | ConvertTo-Json -Depth 6) + [Environment]::NewLine), [Text.UTF8Encoding]::new($false)) [IO.File]::WriteAllText($cursorPath, $cursorKey + [Environment]::NewLine, [Text.UTF8Encoding]::new($false)) $senseEnvironment = @{ SENSE_HTTP_ADDR = "127.0.0.1:$script:senseHTTPPort" SENSE_DB_DRIVER = 'postgres' SENSE_DB_DSN = $senseDSN SENSE_MEDIAMTX_URL = "http://127.0.0.1:$script:productionAPIPort" SENSE_RECONCILE_INTERVAL = '1s' SENSE_RECONCILE_LEASE_DURATION = '30s' SENSE_RECONCILE_OPERATION_TIMEOUT = '20s' SENSE_PROBE_INTERVAL = '1s' SENSE_INSTANCE_ID = 't014-lab' SENSE_METRICS_ENABLED = 'true' SENSE_ORPHAN_SCAN_ENABLED = 'false' SENSE_ONVIF_MODE = 'disabled' SENSE_CONTROL_API_ENABLED = 'true' SENSE_CONTROL_AUTH_FILE = $authPath SENSE_CONTROL_CURSOR_KEY_FILE = $cursorPath SENSE_CREDENTIAL_SYNTHETIC_ONVIF_USERNAME = 't014-local' SENSE_CREDENTIAL_SYNTHETIC_ONVIF_PASSWORD = $localCredential SENSE_CREDENTIAL_SYNTHETIC_RTSP_USERNAME = 't014-local' SENSE_CREDENTIAL_SYNTHETIC_RTSP_PASSWORD = $localCredential } $sense = Start-ManagedProcess 'sense-api' $senseBinary @() $senseEnvironment $managed.Add($sense) Wait-SenseHealth $sense $stage = 'creating 16 disabled devices' $deviceIDs = [Collections.Generic.List[string]]::new() $createEntries = [Collections.Generic.List[object]]::new() foreach ($index in 1..$sourceCount) { $name = 'synthetic-{0:D2}' -f $index $response = Invoke-ControlRequest 'POST' '/api/v1/sites/site-t014/devices' @(201) ([ordered]@{ serial_number = "t014-$name" name = "T-014 source $index" modality = 'video' capabilities = @('video_capture') area_id = 'area-t014' endpoint_ref = "rtsp://127.0.0.1:$script:sourceRTSPPort/$name" credential_ref = 'env://synthetic' desired_state = 'disabled' }) @{ 'Idempotency-Key' = "t014-create-$index-$script:runNonce" } $created = Convert-ResponseJSON $response $deviceIDs.Add([string]$created.id) $createEntries.Add([pscustomobject]@{ DeviceID = [string]$created.id; ETag = Get-ResponseETag $response }) } $stage = 'batch enabling 16 devices' $firstEnableBatch = Invoke-BatchDesiredState 'enabled' @($createEntries) 'enable-1' $firstEnableSeconds = Wait-Convergence 'enabled' $firstEnabledInventory = Get-MediaInventory $script:productionAPIPort $stage = 'rejecting seventeenth enabled device' $overflowResponse = Invoke-ControlRequest 'POST' '/api/v1/sites/site-t014/devices' @(409) ([ordered]@{ serial_number = 't014-overflow-17' name = 'T-014 overflow source' modality = 'video' capabilities = @('video_capture') area_id = 'area-t014' endpoint_ref = "rtsp://127.0.0.1:$script:sourceRTSPPort/synthetic-01" credential_ref = 'env://synthetic' desired_state = 'enabled' }) @{ 'Idempotency-Key' = "t014-overflow-17-$script:runNonce" } $overflowProblem = Convert-ResponseJSON $overflowResponse $overflowCodeProperty = $overflowProblem.PSObject.Properties['code'] if ($null -eq $overflowCodeProperty) { $propertyNames = @($overflowProblem.PSObject.Properties.Name) -join ',' throw "quota response did not contain a stable code field; properties=$propertyNames" } $overflowCode = [string]$overflowCodeProperty.Value $pageAfterOverflow = Get-DevicePage if ($overflowCode -ne 'quota_exceeded' -or @($pageAfterOverflow.items).Count -ne $sourceCount) { throw 'the seventeenth enabled device did not fail closed at the site quota' } $stage = 'batch disabling 16 devices' $disableEntries = Get-DeviceEntries @($deviceIDs) $disableBatch = Invoke-BatchDesiredState 'disabled' $disableEntries 'disable' $disableSeconds = Wait-Convergence 'disabled' $disabledInventory = Get-MediaInventory $script:productionAPIPort $stage = 'batch re-enabling 16 devices' $secondEnableEntries = Get-DeviceEntries @($deviceIDs) $secondEnableBatch = Invoke-BatchDesiredState 'enabled' $secondEnableEntries 'enable-2' $secondEnableSeconds = Wait-Convergence 'enabled' $secondEnabledInventory = Get-MediaInventory $script:productionAPIPort $stage = 'injecting four-source fault' foreach ($index in $faultIndexes) { Stop-ManagedProcess $script:publishers[$index] } $fault = Wait-PartialFault foreach ($index in $faultIndexes) { $script:publishers[$index] = Start-Publisher $index $managed.Add($script:publishers[$index]) } $faultRecoverySeconds = Wait-Convergence 'enabled' $fault.recovery_seconds = $faultRecoverySeconds $fault.final_unconverged = 0 $stage = 'observing stable 16-stream capacity' $observation = [Diagnostics.Stopwatch]::StartNew() $targetSeconds = $ObservationMinutes * 60 $senseCPU = [Collections.Generic.List[double]]::new() $senseWorking = [Collections.Generic.List[double]]::new() $sensePrivate = [Collections.Generic.List[double]]::new() $senseHandles = [Collections.Generic.List[double]]::new() $mediaCPU = [Collections.Generic.List[double]]::new() $mediaWorking = [Collections.Generic.List[double]]::new() $mediaPrivate = [Collections.Generic.List[double]]::new() $mediaHandles = [Collections.Generic.List[double]]::new() $databaseConnections = [Collections.Generic.List[double]]::new() $previousSenseCPU = $sense.Process.TotalProcessorTime.TotalSeconds $previousMediaCPU = $productionMedia.Process.TotalProcessorTime.TotalSeconds $observationSenseCPUStart = $previousSenseCPU $observationMediaCPUStart = $previousMediaCPU $previousSampleAt = 0.0 $initialInventory = Get-MediaInventory $script:productionAPIPort $samples = 0 $maximumUnconverged = 0 while ($observation.Elapsed.TotalSeconds -lt $targetSeconds) { Start-Sleep -Seconds $sampleSeconds Assert-Alive $sourceMedia Assert-Alive $productionMedia Assert-Alive $sense foreach ($publisher in $script:publishers.Values) { Assert-Alive $publisher } $elapsed = $observation.Elapsed.TotalSeconds $delta = $elapsed - $previousSampleAt $senseSample = Get-ProcessResourceSample $sense $previousSenseCPU $delta $mediaSample = Get-ProcessResourceSample $productionMedia $previousMediaCPU $delta $previousSenseCPU = $senseSample.TotalCPU $previousMediaCPU = $mediaSample.TotalCPU $previousSampleAt = $elapsed $senseCPU.Add($senseSample.CPUPercent) $senseWorking.Add($senseSample.WorkingSetMiB) $sensePrivate.Add($senseSample.PrivateMiB) $senseHandles.Add($senseSample.Handles) $mediaCPU.Add($mediaSample.CPUPercent) $mediaWorking.Add($mediaSample.WorkingSetMiB) $mediaPrivate.Add($mediaSample.PrivateMiB) $mediaHandles.Add($mediaSample.Handles) $databaseConnections.Add([double](Get-PostgresScalar "SELECT count(*) FROM pg_stat_activity WHERE datname = '$databaseName'")) $page = Get-DevicePage $inventory = Get-MediaInventory $script:productionAPIPort $unconverged = [int](Get-SenseMetric 'sense_reconcile_devices' '{state="unconverged"}') $maximumUnconverged = [Math]::Max($maximumUnconverged, $unconverged) if (@($page.items).Count -ne $sourceCount -or @($page.items | Where-Object { $_.desired_state -ne 'enabled' -or $_.actual_state -ne 'online' -or -not $_.converged }).Count -ne 0 -or $inventory.Configured -ne $sourceCount -or $inventory.Runtime -ne $sourceCount -or $inventory.Online -ne $sourceCount -or $unconverged -ne 0) { throw 'stability observation detected a non-converged stream' } $samples++ } $observationSeconds = [Math]::Round($observation.Elapsed.TotalSeconds, 1) $finalInventory = Get-MediaInventory $script:productionAPIPort $sense.Process.Refresh() $productionMedia.Process.Refresh() $senseCPUTotal = $sense.Process.TotalProcessorTime.TotalSeconds - $observationSenseCPUStart $mediaCPUTotal = $productionMedia.Process.TotalProcessorTime.TotalSeconds - $observationMediaCPUStart $inboundDelta = [UInt64]($finalInventory.InboundBytes - $initialInventory.InboundBytes) $frameErrorDelta = [UInt64]($finalInventory.FrameErrors - $initialInventory.FrameErrors) $inboundMbps = if ($observationSeconds -gt 0) { ($inboundDelta * 8.0) / $observationSeconds / 1000000.0 } else { 0.0 } $databaseSizeBytes = [Int64](Get-PostgresScalar "SELECT pg_database_size('$databaseName')") $finalUnconverged = [int](Get-SenseMetric 'sense_reconcile_devices' '{state="unconverged"}') if ($finalUnconverged -ne 0 -or $maximumUnconverged -ne 0 -or $samples -lt [Math]::Floor($targetSeconds / $sampleSeconds)) { throw 'formal observation did not satisfy sampling and convergence requirements' } $memoryGiB = 0.0 try { $memoryGiB = (Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory / 1GB } catch {} $gitCommit = (& git -C $script:repoRoot rev-parse HEAD | Out-String).Trim() $success = $true $resultJSON = [ordered]@{ success = $true formal_eligible = $ObservationMinutes -ge 30 scope = 'laboratory_software_baseline' source_count = $sourceCount independent_publishers = $sourceCount fixture = [ordered]@{ width = 640; height = 360; fps = 10; codec = 'H.264'; audio = $false; publish_mode = 'preencoded_copy' } versions = [ordered]@{ repository_commit = $gitCommit sense_sha256 = (Get-FileHash -LiteralPath $senseBinary -Algorithm SHA256).Hash.ToLowerInvariant() mediamtx = $mediaMTXVersion mediamtx_package_sha256 = $mediaMTXSHA256 mediamtx_executable_sha256 = $mediaMTXExecutableSHA256 postgres = $postgresVersion ffmpeg = [string]$ffmpegVersion go = $goVersion } host = [ordered]@{ logical_processors = [Environment]::ProcessorCount; total_memory_gib = [Math]::Round($memoryGiB, 2) } quota = [ordered]@{ configured = 16; accepted_devices = 16; seventeenth_rejected = $true; error_code = 'quota_exceeded' } publishers_ready_seconds = $publisherReadySeconds batch = [ordered]@{ enable_first = $firstEnableBatch disable = $disableBatch enable_second = $secondEnableBatch } convergence_seconds = [ordered]@{ enable_first = $firstEnableSeconds disable = $disableSeconds enable_second = $secondEnableSeconds } configured_paths = [ordered]@{ after_enable_first = $firstEnabledInventory.Configured after_disable = $disabledInventory.Configured after_enable_second = $secondEnabledInventory.Configured } fault_recovery = $fault observation = [ordered]@{ requested_minutes = $ObservationMinutes seconds = $observationSeconds sample_interval_seconds = $sampleSeconds samples = $samples maximum_unconverged = $maximumUnconverged final_unconverged = $finalUnconverged final_online_paths = $finalInventory.Online aggregate_inbound_mbps = [Math]::Round($inboundMbps, 3) inbound_frame_errors = $frameErrorDelta } resources = [ordered]@{ sense = [ordered]@{ cpu_average_percent = [Math]::Round((Get-Average $senseCPU), 3) cpu_peak_percent = [Math]::Round((Get-Maximum $senseCPU), 3) cpu_total_seconds = [Math]::Round($senseCPUTotal, 3) working_set_peak_mib = [Math]::Round((Get-Maximum $senseWorking), 2) private_peak_mib = [Math]::Round((Get-Maximum $sensePrivate), 2) handles_peak = [int](Get-Maximum $senseHandles) } mediamtx = [ordered]@{ cpu_average_percent = [Math]::Round((Get-Average $mediaCPU), 3) cpu_peak_percent = [Math]::Round((Get-Maximum $mediaCPU), 3) cpu_total_seconds = [Math]::Round($mediaCPUTotal, 3) working_set_peak_mib = [Math]::Round((Get-Maximum $mediaWorking), 2) private_peak_mib = [Math]::Round((Get-Maximum $mediaPrivate), 2) handles_peak = [int](Get-Maximum $mediaHandles) } postgres_peak_connections = [int](Get-Maximum $databaseConnections) postgres_database_mib = [Math]::Round($databaseSizeBytes / 1MB, 2) } limitations = @('synthetic_sources', 'no_customer_network', 'no_recording', 'no_downstream_readers', 'no_ai_or_gpu', 'not_a_production_sla') } | ConvertTo-Json -Depth 12 } catch { $failure = "T-014 stage '$stage' failed: $($_.Exception.Message)" } finally { Stop-AllManagedProcesses @($managed) if ($postgresStarted) { & $script:pgCtl '-D' $pgData '-m' 'fast' '-w' 'stop' | Out-Null $postgresStopped = $LASTEXITCODE -eq 0 } if (-not $postgresStarted -or $postgresStopped) { $resolvedSession = [IO.Path]::GetFullPath($script:session) $expectedPrefix = $runtimeResolved.TrimEnd('\') + '\' if (-not $resolvedSession.StartsWith($expectedPrefix, [StringComparison]::OrdinalIgnoreCase) -or [IO.Path]::GetFileName($resolvedSession) -notlike 'session-*') { throw 'refusing to clean an unexpected T-014 session path' } if (Test-Path -LiteralPath $resolvedSession) { Remove-SessionDirectory $resolvedSession } } else { Write-Warning 'Temporary PostgreSQL did not stop; the session directory was retained for manual recovery' } $after5432 = @( Get-NetTCPConnection -State Listen -LocalPort 5432 -ErrorAction SilentlyContinue | Select-Object -ExpandProperty OwningProcess -Unique | Sort-Object ) if (($existing5432 -join ',') -ne ($after5432 -join ',')) { throw 'the existing PostgreSQL listener on port 5432 changed during T-014' } } if (-not $success) { throw $failure } Write-Result $resultJSON