fix(android): correct camera frame rotation and recorder timestamps

Rotate camera frames before encoding while accounting for Camera2 texture transforms. Align camera timestamps with MediaRecorder and cover capture duration, tracks and rotation with regression tests. Include LibVLC ProGuard rules and update documentation and changelog.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

src/PostIt/PostIt.Tests/AndroidCameraFrameRotationTestActivity.cs

@@ -0,0 +1,147 @@
+#if ANDROID
+using System;
+using System.IO;
+using System.Runtime.Versioning;
+using System.Threading;
+using System.Threading.Tasks;
+using Android.App;
+using Android.Graphics;
+using Android.Media;
+using Android.OS;
+using PostIt.Android.Services;
+using Path = System.IO.Path;
+
+namespace PostIt.Tests;
+
+/// <summary>Debug-only synthetic encoder test; never opens the camera or contacts the API.</summary>
+[Activity(Name = "fr.pschneider.postit.CameraFrameRotationTestActivity", Exported = true)]
+[SupportedOSPlatform("android26.0")]
+public sealed class AndroidCameraFrameRotationTestActivity : Activity
+{
+    protected override async void OnCreate(Bundle? state)
+    {
+        base.OnCreate(state);
+        var resultPath = Path.Combine(FilesDir!.AbsolutePath, "camera-rotation-test-result.txt");
+        File.WriteAllText(resultPath, "RUNNING");
+        try
+        {
+            foreach (var rotation in new[] { 0, 90, 180, 270 })
+                await VerifyRotationAsync(rotation);
+            File.WriteAllText(resultPath, "PASS\nrotations=0,90,180,270\nformat=MPEG-TS\nsource=synthetic");
+        }
+        catch (Exception ex)
+        {
+            File.WriteAllText(resultPath, $"FAIL\n{ex}");
+        }
+        finally { Finish(); }
+    }
+
+    private async Task VerifyRotationAsync(int rotation)
+    {
+        const int width = 640;
+        const int height = 480;
+        var outputWidth = rotation is 90 or 270 ? height : width;
+        var outputHeight = rotation is 90 or 270 ? width : height;
+        var path = Path.Combine(FilesDir!.AbsolutePath, $"camera-rotation-{rotation}.ts");
+        void Stage(string stage) => File.WriteAllText(
+            Path.Combine(FilesDir.AbsolutePath, "camera-rotation-test-result.txt"), $"RUNNING\nrotation={rotation}\nstage={stage}");
+        Stage("prepare");
+#pragma warning disable CS0618
+        using var recorder = OperatingSystem.IsAndroidVersionAtLeast(31)
+            ? new MediaRecorder(this) : new MediaRecorder();
+#pragma warning restore CS0618
+        recorder.SetVideoSource(VideoSource.Surface);
+        recorder.SetOutputFormat(OutputFormat.Mpeg2Ts);
+        recorder.SetVideoSize(outputWidth, outputHeight);
+        recorder.SetVideoFrameRate(30);
+        recorder.SetVideoEncodingBitRate(1200000);
+        recorder.SetVideoEncoder(VideoEncoder.H264);
+        recorder.SetOutputFile(path);
+        recorder.Prepare();
+        using var encoderSurface = recorder.Surface
+            ?? throw new InvalidOperationException("Encoder surface unavailable.");
+        Exception? failure = null;
+        var stopping = 0;
+        Stage("initialize EGL");
+        using (var rotator = await AndroidCameraFrameRotator.CreateAsync(
+            encoderSurface, width, height, rotation, error =>
+            {
+                if (Volatile.Read(ref stopping) != 0) return;
+                failure = error;
+                File.WriteAllText(Path.Combine(FilesDir.AbsolutePath, "camera-rotation-test-result.txt"), $"FAIL\n{error}");
+            }))
+        {
+            Stage("start recorder");
+            recorder.Start();
+            Task? stop = null;
+            try
+            {
+                await rotator.StartAsync();
+                using var paint = new Paint();
+                Color[] colors = [Color.Red, Color.Green, Color.Blue, Color.Yellow];
+                Stage("draw frames");
+                for (var frame = 0; frame < 90; frame++)
+                {
+                    if (failure is not null) throw failure;
+                    if (frame == 45)
+                    {
+                        Volatile.Write(ref stopping, 1);
+                        stop = Task.Run(recorder.Stop);
+                    }
+                    if (stop?.IsCompleted == true) break;
+                    var canvas = rotator.CameraSurface.LockCanvas(null)
+                        ?? throw new InvalidOperationException("Synthetic camera canvas unavailable.");
+                    try
+                    {
+                        for (var i = 0; i < colors.Length; i++)
+                        {
+                            paint.Color = colors[i];
+                            var x = i % 2 * width / 2;
+                            var y = i / 2 * height / 2;
+                            canvas.DrawRect(x, y, x + width / 2, y + height / 2, paint);
+                        }
+                    }
+                    finally
+                    {
+                        rotator.CameraSurface.UnlockCanvasAndPost(canvas);
+                        canvas.Dispose();
+                    }
+                    await Task.Delay(34);
+                }
+                if (failure is not null) throw failure;
+                Stage("wait for recorder stop");
+                await (stop ?? throw new InvalidOperationException("Recorder stop was not requested."))
+                    .WaitAsync(TimeSpan.FromSeconds(10));
+            }
+            finally
+            {
+                Stage("dispose EGL");
+                rotator.Dispose();
+            }
+        }
+        Stage("decode");
+        using var retriever = new MediaMetadataRetriever();
+        retriever.SetDataSource(path);
+        using var bitmap = retriever.GetFrameAtTime(0, Option.ClosestSync)
+            ?? throw new InvalidOperationException($"Cannot decode rotation {rotation}.");
+        if (bitmap.Width != outputWidth || bitmap.Height != outputHeight)
+            throw new InvalidOperationException($"Rotation {rotation}: dimensions {bitmap.Width}x{bitmap.Height}, expected {outputWidth}x{outputHeight}.");
+        Color[] expected = rotation switch
+        {
+            90 => [Color.Blue, Color.Red, Color.Yellow, Color.Green],
+            180 => [Color.Yellow, Color.Blue, Color.Green, Color.Red],
+            270 => [Color.Green, Color.Yellow, Color.Red, Color.Blue],
+            _ => [Color.Red, Color.Green, Color.Blue, Color.Yellow]
+        };
+        for (var i = 0; i < expected.Length; i++)
+        {
+            var actual = new Color(bitmap.GetPixel((i % 2 * 2 + 1) * outputWidth / 4,
+                (i / 2 * 2 + 1) * outputHeight / 4));
+            var target = expected[i];
+            if (Math.Abs(actual.R - target.R) > 40 || Math.Abs(actual.G - target.G) > 40 || Math.Abs(actual.B - target.B) > 40)
+                throw new InvalidOperationException($"Rotation {rotation}, quadrant {i}: {actual}, expected {target}.");
+        }
+        File.Delete(path);
+    }
+}
+#endif

src/PostIt/PostIt.Tests/AndroidPodcastCaptureTestActivity.cs

@@ -1,11 +1,13 @@
 #if ANDROID
 using System;
 using System.IO;
+using System.Linq;
 using System.Runtime.Versioning;
 using System.Threading;
 using System.Threading.Tasks;
 using Android.App;
 using Android.Content;
+using Android.Media;
 using Android.OS;
 using Avalonia.Threading;
 using PostIt.Android;
@@ -34,7 +36,7 @@ public sealed class AndroidPodcastCaptureTestActivity : Activity
         try
         {
             await RunAsync(result);
-            File.WriteAllText(resultPath, $"PASS\nsegments={result.GetInt("segments")}\ndirectory={result.GetString("directory")}\nbackground={result.GetString("background")}");
+            File.WriteAllText(resultPath, $"PASS\nsegments={result.GetInt("segments")}\ndurationSeconds={result.GetDouble("durationSeconds")}\naudio={result.GetBoolean("audio")}\ndirectory={result.GetString("directory")}\nbackground={result.GetString("background")}\norientation={result.GetString("orientation")}\ntextureTransform={result.GetString("textureTransform")}");
         }
         catch (Exception ex)
         {
@@ -68,6 +70,11 @@ public sealed class AndroidPodcastCaptureTestActivity : Activity
             await Task.Delay(9000);
             var after = SegmentCount(directory);
             if (capture.Completion.IsFaulted) await capture.Completion;
+            if (capture is AndroidPodcastCapture androidCapture)
+            {
+                result.PutString("orientation", androidCapture.OrientationDiagnostics);
+                result.PutString("textureTransform", androidCapture.TextureTransform);
+            }
             if (after <= before) throw new InvalidOperationException($"Capture did not continue in background: {before} -> {after} segments.");
             if (_notificationStop)
             {
@@ -81,7 +88,25 @@ public sealed class AndroidPodcastCaptureTestActivity : Activity
             await capture.Completion.WaitAsync(TimeSpan.FromSeconds(15));
             var segments = HlsCapturePlaylist.Parse(await File.ReadAllTextAsync(Path.Combine(directory, "index.m3u8")));
             if (segments.Count < 3) throw new InvalidOperationException("Insufficient capture segments.");
+            var duration = segments.Sum(segment => segment.DurationSeconds);
+            if (duration < 8 || duration > 16)
+                throw new InvalidOperationException($"Expected 8-16 seconds of capture, got {duration}.");
+            using var extractor = new MediaExtractor();
+            extractor.SetDataSource(Path.Combine(directory, segments[0].FileName));
+            var video = false;
+            var audio = false;
+            for (var track = 0; track < extractor.TrackCount; track++)
+            {
+                using var format = extractor.GetTrackFormat(track);
+                var mime = format.GetString(MediaFormat.KeyMime);
+                video |= mime == "video/avc";
+                audio |= mime == "audio/mp4a-latm";
+            }
+            if (!video || (_microphone && !audio))
+                throw new InvalidOperationException($"Missing capture tracks: H.264={video}, AAC={audio}.");
             result.PutInt("segments", segments.Count);
+            result.PutDouble("durationSeconds", duration);
+            result.PutBoolean("audio", audio);
             result.PutString("directory", directory);
             result.PutString("background", $"segments {before} -> {after}");
         }

src/PostIt/PostIt.Tests/CameraFrameClockTests.cs

@@ -0,0 +1,37 @@
+using PostIt.Android.Services;
+
+namespace PostIt.Tests;
+
+public sealed class CameraFrameClockTests
+{
+    [Theory]
+    [InlineData(4_213_000_000_000L)]
+    [InlineData(0L)]
+    [InlineData(-1_000_000_000L)]
+    public void Aligns_camera_epoch_to_encoder_and_preserves_frame_intervals(long cameraOffset)
+    {
+        var clock = new CameraFrameClock();
+        const long now = 20_000_000_000_000L;
+        var camera = now + cameraOffset;
+        Assert.Equal(now, clock.GetPresentationTime(camera, now));
+        Assert.Equal(now + 33_333_333, clock.GetPresentationTime(camera + 33_333_333, now + 40_000_000));
+        Assert.Equal(now + 100_000_000, clock.GetPresentationTime(camera + 100_000_000, now + 110_000_000));
+    }
+
+    [Fact]
+    public void Does_not_submit_the_same_camera_image_twice()
+    {
+        var clock = new CameraFrameClock();
+        Assert.Equal(100, clock.GetPresentationTime(1000, 100));
+        Assert.Null(clock.GetPresentationTime(1000, 110));
+        Assert.Equal(120, clock.GetPresentationTime(1020, 130));
+    }
+
+    [Fact]
+    public void Reports_a_camera_clock_that_goes_backwards()
+    {
+        var clock = new CameraFrameClock();
+        clock.GetPresentationTime(1000, 100);
+        Assert.Throws<InvalidOperationException>(() => clock.GetPresentationTime(999, 110));
+    }
+}

src/PostIt/PostIt.Tests/CameraFrameOrientationTests.cs

@@ -0,0 +1,61 @@
+using PostIt.Android.Services;
+
+namespace PostIt.Tests;
+
+public sealed class CameraFrameOrientationTests
+{
+    [Theory]
+    [InlineData(90, 0, false, 90)]
+    [InlineData(90, 90, false, 0)]
+    [InlineData(90, 180, false, 270)]
+    [InlineData(90, 270, false, 180)]
+    [InlineData(270, 0, true, 270)]
+    [InlineData(270, 90, true, 0)]
+    [InlineData(270, 180, true, 90)]
+    [InlineData(270, 270, true, 180)]
+    [InlineData(0, 90, false, 270)]
+    [InlineData(180, 270, true, 90)]
+    public void Uses_sensor_display_and_lens_facing(int sensor, int display, bool front, int expected)
+        => Assert.Equal(expected, CameraFrameOrientation.GetRotation(sensor, display, front));
+
+    [Theory]
+    [InlineData(0, new float[] { 0, 0, 1, 0, 0, 1, 1, 1 })]
+    [InlineData(90, new float[] { 1, 0, 1, 1, 0, 0, 0, 1 })]
+    [InlineData(180, new float[] { 1, 1, 0, 1, 1, 0, 0, 0 })]
+    [InlineData(270, new float[] { 0, 1, 0, 0, 1, 1, 1, 0 })]
+    public void Maps_output_corners_to_input_for_clockwise_rotation(int rotation, float[] expected)
+        => Assert.Equal(expected, CameraFrameOrientation.GetTextureCoordinates(rotation));
+
+    [Theory]
+    [InlineData(90, 0, -1, 0)]
+    [InlineData(270, 0, 1, 0)]
+    [InlineData(180, -1, 0, 0)]
+    [InlineData(90, 1, 0, 90)]
+    [InlineData(270, 1, 0, 270)]
+    [InlineData(0, 0, -1, 270)]
+    [InlineData(180, 0, -1, 90)]
+    public void Does_not_repeat_rotation_already_in_surface_texture(
+        int desired, float x, float y, int expected)
+    {
+        float[] matrix = [x, y, 0, 0, y, -x, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1];
+        Assert.Equal(expected, CameraFrameOrientation.GetRemainingRotation(desired, matrix));
+    }
+
+    [Fact]
+    public void Recognizes_the_measured_rear_camera_transform_even_with_cropping()
+    {
+        float[] matrix = [0, -0.9f, 0, 0, -0.8f, 0, 0, 0, 0, 0, 1, 0, 0.9f, 0.95f, 0, 1];
+        Assert.Equal(0, CameraFrameOrientation.GetRemainingRotation(90, matrix));
+    }
+
+    [Theory]
+    [InlineData(-90)]
+    [InlineData(45)]
+    [InlineData(360)]
+    public void Rejects_invalid_angles(int rotation)
+    {
+        Assert.Throws<ArgumentOutOfRangeException>(() => CameraFrameOrientation.GetRotation(rotation, 0, false));
+        Assert.Throws<ArgumentOutOfRangeException>(() => CameraFrameOrientation.GetRotation(90, rotation, false));
+        Assert.Throws<ArgumentOutOfRangeException>(() => CameraFrameOrientation.GetTextureCoordinates(rotation));
+    }
+}

src/PostIt/PostIt.Tests/PostIt.Tests.csproj

@@ -25,5 +25,8 @@
   <ItemGroup>
     <Using Include="Xunit" />
   </ItemGroup>
-  <ItemGroup></ItemGroup>
+  <ItemGroup>
+    <Compile Include="..\PostIt.Android\Services\CameraFrameOrientation.cs" Link="Android\CameraFrameOrientation.cs" />
+    <Compile Include="..\PostIt.Android\Services\CameraFrameClock.cs" Link="Android\CameraFrameClock.cs" />
+  </ItemGroup>
 </Project>