[CmdletBinding()] param( [Parameter(Mandatory = $true)] [string]$CameraEnv, [string]$RuntimeRoot = (Join-Path ([IO.Path]::GetTempPath()) 'yovision-t006'), [ValidateRange(1, 1440)] [int]$ObservationMinutes = 30, [switch]$KeepSession ) $ErrorActionPreference = 'Stop' $ProgressPreference = 'SilentlyContinue' function Read-EnvFile([string]$Path) { $result = @{} Get-Content -LiteralPath $Path | ForEach-Object { $line = $_.Trim() if (-not $line -or $line.StartsWith('#') -or -not $line.Contains('=')) { return } $parts = $line -split '=', 2 $value = $parts[1].Trim() if ($value.Length -ge 2 -and (($value.StartsWith('"') -and $value.EndsWith('"')) -or ($value.StartsWith("'") -and $value.EndsWith("'")))) { $value = $value.Substring(1, $value.Length - 2) } $result[$parts[0].Trim().ToLowerInvariant()] = $value } return $result } function Require-Keys([hashtable]$Config, [string[]]$Keys) { foreach ($key in $Keys) { if (-not $Config.ContainsKey($key) -or [string]::IsNullOrWhiteSpace($Config[$key])) { throw "camera environment is missing required key: $key" } } } function Assert-PortFree([int]$Port) { $client = [Net.Sockets.TcpClient]::new() try { $task = $client.ConnectAsync('127.0.0.1', $Port) if ($task.Wait(250) -and $client.Connected) { throw "required local port is already in use: $Port" } } catch [AggregateException] { return } catch [Net.Sockets.SocketException] { return } finally { $client.Dispose() } } function Wait-Port([int]$Port, [int]$TimeoutSeconds = 30) { $watch = [Diagnostics.Stopwatch]::StartNew() while ($watch.Elapsed.TotalSeconds -lt $TimeoutSeconds) { $client = [Net.Sockets.TcpClient]::new() try { $task = $client.ConnectAsync('127.0.0.1', $Port) if ($task.Wait(500) -and $client.Connected) { return } } catch { } finally { $client.Dispose() } Start-Sleep -Milliseconds 250 } throw "local port did not become ready: $Port" } function Start-ManagedProcess( [string]$Name, [string]$FilePath, [string[]]$Arguments, [hashtable]$Environment = @{} ) { $start = [Diagnostics.ProcessStartInfo]::new() $start.FileName = $FilePath $start.WorkingDirectory = $session $start.UseShellExecute = $false $start.CreateNoWindow = $true $start.RedirectStandardOutput = $true $start.RedirectStandardError = $true foreach ($argument in $Arguments) { $start.ArgumentList.Add($argument) } foreach ($entry in $Environment.GetEnumerator()) { $start.Environment[$entry.Key] = [string]$entry.Value } $process = [Diagnostics.Process]::new() $process.StartInfo = $start if (-not $process.Start()) { throw "failed to start process: $Name" } return [pscustomobject]@{ Name = $Name Process = $process Stdout = $process.StandardOutput.ReadToEndAsync() Stderr = $process.StandardError.ReadToEndAsync() StartedAt = [DateTimeOffset]::UtcNow } } function Stop-ManagedProcess($Managed) { if ($null -eq $Managed -or $null -eq $Managed.Process) { return } if (-not $Managed.Process.HasExited) { # Every fixture is started as a leaf process. Killing only that exact # process avoids Windows process-tree edge cases during fault tests. $Managed.Process.Kill() $Managed.Process.WaitForExit(10000) | Out-Null } } function Assert-Alive($Managed) { if ($null -eq $Managed -or $Managed.Process.HasExited) { $code = if ($null -eq $Managed) { 'not-started' } else { $Managed.Process.ExitCode } throw "required process exited: $($Managed.Name), code=$code" } } function Invoke-LabStatus { $lastDiagnostic = '' foreach ($attempt in 1..5) { $raw = @(& $labBinary status -db $databaseDSN 2>&1) if ($LASTEXITCODE -eq 0) { try { return (($raw -join "`n") | ConvertFrom-Json) } catch { $lastDiagnostic = 'invalid JSON response' } } else { $lastDiagnostic = (($raw -join "`n") -split "`r?`n" | Select-Object -Last 2) -join ' | ' } Start-Sleep -Milliseconds 250 } throw "sense-lab status failed after retries: $lastDiagnostic" } function Wait-Converged([int]$TimeoutSeconds = 180) { $watch = [Diagnostics.Stopwatch]::StartNew() while ($watch.Elapsed.TotalSeconds -lt $TimeoutSeconds) { try { $snapshot = Invoke-LabStatus if ([int]$snapshot.total -eq 5 -and [int]$snapshot.unconverged -eq 0) { return [Math]::Round($watch.Elapsed.TotalSeconds, 1) } } catch { } Start-Sleep -Seconds 1 } throw 'five devices did not converge before timeout' } function Wait-DeviceState([string]$ID, [string]$State, [int]$TimeoutSeconds = 90) { $watch = [Diagnostics.Stopwatch]::StartNew() while ($watch.Elapsed.TotalSeconds -lt $TimeoutSeconds) { try { $snapshot = Invoke-LabStatus $match = @($snapshot.devices | Where-Object { $_.id -eq $ID }) if ($match.Count -eq 1 -and $match[0].actual_state -eq $State) { return [Math]::Round($watch.Elapsed.TotalSeconds, 1) } } catch { } Start-Sleep -Seconds 1 } throw "device did not reach expected state: $ID/$State" } function Wait-SenseHealth($Managed, [int]$TimeoutSeconds = 30) { $watch = [Diagnostics.Stopwatch]::StartNew() while ($watch.Elapsed.TotalSeconds -lt $TimeoutSeconds) { if ($Managed.Process.HasExited) { $output = @( $Managed.Stdout.GetAwaiter().GetResult() $Managed.Stderr.GetAwaiter().GetResult() ) -join "`n" $summary = (($output -split "`r?`n") | Where-Object { $_ } | Select-Object -Last 3) -join ' | ' throw "Sense exited before health check, code=$($Managed.Process.ExitCode), output=$summary" } try { $response = Invoke-RestMethod -Method Get -Uri 'http://127.0.0.1:18080/healthz' -TimeoutSec 2 -NoProxy if ($response.status -eq 'ok') { return } } catch { } Start-Sleep -Milliseconds 500 } $listeners = @(Get-NetTCPConnection -State Listen -OwningProcess $Managed.Process.Id -ErrorAction SilentlyContinue | ForEach-Object { "$($_.LocalAddress):$($_.LocalPort)" }) throw "Sense health endpoint did not become ready; process listeners=$($listeners -join ',')" } function Start-Publisher([int]$Index) { return Start-ManagedProcess "publisher-$Index" $ffmpeg @( '-hide_banner', '-loglevel', 'warning', '-re', '-f', 'lavfi', '-i', "testsrc2=size=640x360:rate=10", '-c:v', 'libx264', '-preset', 'ultrafast', '-tune', 'zerolatency', '-pix_fmt', 'yuv420p', '-g', '10', '-an', '-f', 'rtsp', '-rtsp_transport', 'tcp', "rtsp://127.0.0.1:8555/synthetic-$Index" ) } function Start-Proxy { return Start-ManagedProcess 'real-camera-network-proxy' $proxyBinary @( '-listen', '127.0.0.1:10554', '-upstream', "$($camera.host):$($camera.rtspport)" ) } function Start-Sense { $environment = @{ SENSE_HTTP_ADDR = '127.0.0.1:18080' SENSE_DB_DSN = $databaseDSN SENSE_MEDIAMTX_URL = 'http://127.0.0.1:9997' SENSE_RECONCILE_INTERVAL = '1s' SENSE_PROBE_INTERVAL = '1s' SENSE_ONVIF_MODE = 'standard' SENSE_ONVIF_RTSP_REWRITE_HOST = '127.0.0.1' SENSE_ONVIF_RTSP_REWRITE_PORT = '10554' SENSE_ONVIF_RTSP_STRIP_QUERY = 'true' SENSE_CREDENTIAL_CAMERA_ONVIF_USERNAME = $camera.onvifuser SENSE_CREDENTIAL_CAMERA_ONVIF_PASSWORD = $camera.onvifpwd SENSE_CREDENTIAL_CAMERA_RTSP_USERNAME = $camera.username SENSE_CREDENTIAL_CAMERA_RTSP_PASSWORD = $camera.password } return Start-ManagedProcess -Name 'sense-api' -FilePath $senseBinary -Arguments @() -Environment $environment } function Start-ProductionMediaMTX { return Start-ManagedProcess 'mediamtx-production' $mediaMTX @($productionConfig) } $repoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path $cameraPath = (Resolve-Path -LiteralPath $CameraEnv).Path $camera = Read-EnvFile $cameraPath Require-Keys $camera @('host', 'username', 'password', 'rtspport', 'onvif', 'onvifuser', 'onvifpwd') foreach ($port in 8554, 8555, 9997, 10554, 18080) { Assert-PortFree $port } New-Item -ItemType Directory -Path $RuntimeRoot -Force | Out-Null $mediaDirectory = Join-Path $RuntimeRoot 'mediamtx-v1.19.3' $mediaMTX = Join-Path $mediaDirectory 'mediamtx.exe' if (-not (Test-Path -LiteralPath $mediaMTX)) { $zip = Join-Path $RuntimeRoot 'mediamtx_v1.19.3_windows_amd64.zip' Invoke-WebRequest 'https://github.com/bluenviron/mediamtx/releases/download/v1.19.3/mediamtx_v1.19.3_windows_amd64.zip' -OutFile $zip $actualHash = (Get-FileHash -LiteralPath $zip -Algorithm SHA256).Hash.ToLowerInvariant() if ($actualHash -ne '5d82148d1032a6a190d9909a2997d9989457aaadf49af87dd02cd4512d31bebe') { throw 'MediaMTX checksum mismatch' } New-Item -ItemType Directory -Path $mediaDirectory -Force | Out-Null Expand-Archive -LiteralPath $zip -DestinationPath $mediaDirectory -Force } $ffmpeg = (Get-Command ffmpeg -ErrorAction Stop).Source $session = Join-Path $RuntimeRoot ('session-' + [Guid]::NewGuid().ToString('N')) New-Item -ItemType Directory -Path $session | Out-Null $senseBinary = Join-Path $session 'sense-api.exe' $labBinary = Join-Path $session 'sense-lab.exe' $proxyBinary = Join-Path $session 'rtsp-fault-proxy.exe' $productionConfig = Join-Path $session 'mediamtx-production.yml' $syntheticConfig = Join-Path $session 'mediamtx-synthetic.yml' [IO.File]::Copy((Join-Path $repoRoot 'Sense\deploy\mediamtx.yml'), $productionConfig) [IO.File]::Copy((Join-Path $repoRoot 'Sense\deploy\mediamtx-synthetic.yml'), $syntheticConfig) & go -C (Join-Path $repoRoot 'Sense') build -o $senseBinary ./cmd/sense-api if ($LASTEXITCODE -ne 0) { throw 'build sense-api failed' } & go -C (Join-Path $repoRoot 'Sense') build -o $labBinary ./cmd/sense-lab if ($LASTEXITCODE -ne 0) { throw 'build sense-lab failed' } & go -C (Join-Path $repoRoot 'Sense') build -o $proxyBinary ./cmd/rtsp-fault-proxy if ($LASTEXITCODE -ne 0) { throw 'build RTSP fault proxy failed' } $databasePath = Join-Path $session 'sense.db' $databaseDSN = 'file:' + $databasePath.Replace('\', '/') $manifestPath = Join-Path $session 'manifest.json' $onvifEndpoint = if ($camera.onvif -match '^https?://') { $camera.onvif } else { "http://$($camera.host):$($camera.onvif)/onvif/device_service" } $devices = @( [ordered]@{ id = 'camera-real'; tenant_id = 'lab'; site_id = 'site'; serial_number = 'e9ed6a555ae0'; name = 'Approved real camera'; capabilities = @('video_capture', 'audio_capture'); endpoint_ref = $onvifEndpoint; credential_ref = 'env://camera'; path_name = 'sense/lab/site/camera-real' } ) foreach ($index in 1..4) { $devices += [ordered]@{ id = "synthetic-$index"; tenant_id = 'lab'; site_id = 'site'; serial_number = "synthetic-$index"; name = "Synthetic source $index"; capabilities = @('video_capture'); endpoint_ref = "rtsp://127.0.0.1:8555/synthetic-$index"; credential_ref = ''; path_name = "sense/lab/site/synthetic-$index" } } [ordered]@{ site = [ordered]@{ tenant_id = 'lab'; id = 'site'; name = 'T-006 Lab'; max_video_channels = 16 } devices = $devices } | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $manifestPath -Encoding UTF8 $managed = [Collections.Generic.List[object]]::new() $publishers = @{} $events = [Collections.Generic.List[object]]::new() $observationSamples = 0 $maxUnconverged = 0 $success = $false $stage = 'starting fixtures' $failureMessage = $null try { $sourceMedia = Start-ManagedProcess 'mediamtx-synthetic' $mediaMTX @($syntheticConfig) $managed.Add($sourceMedia) Wait-Port 8555 $productionMedia = Start-ProductionMediaMTX $managed.Add($productionMedia) Wait-Port 9997 foreach ($index in 1..4) { $publishers[$index] = Start-Publisher $index $managed.Add($publishers[$index]) } $stage = 'checking synthetic publishers' Start-Sleep -Seconds 3 foreach ($publisher in $publishers.Values) { Assert-Alive $publisher } $proxy = Start-Proxy $managed.Add($proxy) Wait-Port 10554 & $labBinary seed -db $databaseDSN -manifest $manifestPath | Out-Null if ($LASTEXITCODE -ne 0) { throw 'seed device ledger failed' } $stage = 'initial convergence' $sense = Start-Sense $managed.Add($sense) Wait-SenseHealth $sense $initialSeconds = Wait-Converged $events.Add([ordered]@{ event = 'initial_convergence'; seconds = $initialSeconds; unconverged = 0 }) $stage = 'real camera network recovery' Stop-ManagedProcess $proxy $offlineSeconds = Wait-DeviceState 'camera-real' 'offline' $proxy = Start-Proxy $managed.Add($proxy) Wait-Port 10554 $recoverySeconds = Wait-Converged $events.Add([ordered]@{ event = 'real_camera_network'; offline_detect_seconds = $offlineSeconds; recovery_seconds = $recoverySeconds; unconverged = 0 }) $stage = 'synthetic publisher recovery' Stop-ManagedProcess $publishers[2] $offlineSeconds = Wait-DeviceState 'synthetic-2' 'offline' $publishers[2] = Start-Publisher 2 $managed.Add($publishers[2]) $recoverySeconds = Wait-Converged $events.Add([ordered]@{ event = 'synthetic_publisher'; offline_detect_seconds = $offlineSeconds; recovery_seconds = $recoverySeconds; unconverged = 0 }) $stage = 'Sense restart recovery' Stop-ManagedProcess $sense $restartWatch = [Diagnostics.Stopwatch]::StartNew() $sense = Start-Sense $managed.Add($sense) Wait-SenseHealth $sense $recoverySeconds = Wait-Converged $events.Add([ordered]@{ event = 'sense_restart'; process_ready_seconds = [Math]::Round($restartWatch.Elapsed.TotalSeconds, 1); recovery_seconds = $recoverySeconds; unconverged = 0 }) $stage = 'MediaMTX restart recovery' Stop-ManagedProcess $productionMedia Start-Sleep -Seconds 3 $productionMedia = Start-ProductionMediaMTX $managed.Add($productionMedia) Wait-Port 9997 $recoverySeconds = Wait-Converged 240 $events.Add([ordered]@{ event = 'mediamtx_restart'; recovery_seconds = $recoverySeconds; unconverged = 0 }) $stage = 'checking MediaMTX path count' $configured = Invoke-RestMethod -Method Get -Uri 'http://127.0.0.1:9997/v3/config/paths/list' -TimeoutSec 5 -NoProxy if ([int]$configured.itemCount -ne 5) { throw "expected 5 MediaMTX paths, got $($configured.itemCount)" } $stage = 'stability observation' $observation = [Diagnostics.Stopwatch]::StartNew() $targetSeconds = $ObservationMinutes * 60 while ($observation.Elapsed.TotalSeconds -lt $targetSeconds) { Assert-Alive $sourceMedia Assert-Alive $productionMedia Assert-Alive $sense Assert-Alive $proxy foreach ($publisher in $publishers.Values) { Assert-Alive $publisher } $snapshot = Invoke-LabStatus $observationSamples++ $maxUnconverged = [Math]::Max($maxUnconverged, [int]$snapshot.unconverged) if ([int]$snapshot.total -ne 5 -or [int]$snapshot.unconverged -ne 0) { throw "observation detected unconverged devices: $($snapshot.unconverged)" } Start-Sleep -Seconds 10 } $observationSeconds = [Math]::Round($observation.Elapsed.TotalSeconds, 1) $final = Invoke-LabStatus $success = $true [ordered]@{ success = $true mediamtx_version = 'v1.19.3' mediamtx_sha256 = '5d82148d1032a6a190d9909a2997d9989457aaadf49af87dd02cd4512d31bebe' source_count = 5 synthetic_publishers = 4 configured_paths = [int]$configured.itemCount recovery = $events observation_seconds = $observationSeconds observation_samples = $observationSamples maximum_unconverged = $maxUnconverged final_unconverged = [int]$final.unconverged } | ConvertTo-Json -Depth 8 } catch { $failureMessage = "T-006 stage '$stage' failed: $($_.Exception.Message)" } finally { foreach ($item in @($managed)) { Stop-ManagedProcess $item } if (-not $KeepSession) { Get-ChildItem -LiteralPath $session -Recurse -Force -ErrorAction SilentlyContinue | ForEach-Object { $_.Attributes = 'Normal' } if (Test-Path -LiteralPath $session) { (Get-Item -LiteralPath $session -Force).Attributes = 'Directory' Remove-Item -LiteralPath $session -Recurse -ErrorAction SilentlyContinue } } if (-not $success) { Write-Warning 'T-006 integration did not complete; no success evidence was emitted.' } } if ($failureMessage) { throw $failureMessage }