fix(android): correct camera frame rotation and recorder timestamps
All checks were successful
Dotnet build and test / build (push) Successful in 9m59s
Forgejo Release / release (push) Successful in 9m41s

Rotate camera frames before encoding while accounting for Camera2 texture transforms. Align camera timestamps with MediaRecorder and cover capture duration, tracks and rotation with regression tests. Include LibVLC ProGuard rules and update documentation and changelog.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Paul Schneider 2026-10-24 23:09:37 +01:00
commit ed75f06fb2
Signed by: notazof
GPG key ID: AD4B9B545FBD7D12
14 changed files with 792 additions and 8 deletions

View file

@ -31,6 +31,7 @@
</ItemGroup>
<ItemGroup Condition="'$(Configuration)' == 'Debug'">
<Compile Include="..\PostIt.Tests\AndroidPodcastCaptureTestActivity.cs" Link="Tests\AndroidPodcastCaptureTestActivity.cs" />
<Compile Include="..\PostIt.Tests\AndroidCameraFrameRotationTestActivity.cs" Link="Tests\AndroidCameraFrameRotationTestActivity.cs" />
</ItemGroup>
<!-- This is an Avalonia app, not a MAUI app: exclude the .NET MAUI VS Code
extension's injected HotReload agent (references Microsoft.Maui.*).

View file

@ -0,0 +1,308 @@
using System;
using System.Threading.Tasks;
using Android.Graphics;
using Android.Opengl;
using Android.OS;
using Android.Views;
using Java.Nio;
namespace PostIt.Android.Services;
internal sealed class AndroidCameraFrameRotator : IDisposable
{
private const int ExternalTexture = 0x8D65;
private const int RecordableAndroid = 0x3142;
private const string VertexShader = """
attribute vec2 position;
attribute vec2 textureCoordinate;
uniform mat4 textureMatrix;
varying vec2 uv;
void main() {
gl_Position = vec4(position, 0.0, 1.0);
uv = (textureMatrix * vec4(textureCoordinate, 0.0, 1.0)).xy;
}
""";
private const string FragmentShader = """
#extension GL_OES_EGL_image_external : require
precision mediump float;
uniform samplerExternalOES cameraTexture;
varying vec2 uv;
void main() {
gl_FragColor = texture2D(cameraTexture, uv);
}
""";
private readonly HandlerThread _thread = new("PostIt.CameraRotation");
private readonly Handler _handler;
private readonly Action<Exception> _failed;
private readonly float[] _textureMatrix = new float[16];
private readonly CameraFrameClock _clock = new();
private EGLDisplay? _display;
private EGLContext? _context;
private EGLSurface? _window;
private SurfaceTexture? _texture;
private FrameListener? _listener;
private Surface? _cameraSurface;
private FloatBuffer? _vertices;
private FloatBuffer? _coordinates;
private int _program;
private int _textureId;
private int _matrixLocation;
private bool _rendering;
private bool _disposed;
private int _rotation;
private int? _remainingRotation;
internal string? FirstTextureTransform { get; private set; }
internal Surface CameraSurface => _cameraSurface
?? throw new InvalidOperationException("Surface de rotation camera indisponible.");
private AndroidCameraFrameRotator(Action<Exception> failed)
{
_failed = failed;
_thread.Start();
_handler = new Handler(_thread.Looper!);
}
internal static async Task<AndroidCameraFrameRotator> CreateAsync(
Surface encoderSurface, int width, int height, int rotation, Action<Exception> failed)
{
var rotator = new AndroidCameraFrameRotator(failed);
try
{
await rotator.InvokeAsync(() => rotator.Initialize(encoderSurface, width, height, rotation));
return rotator;
}
catch
{
rotator.Dispose();
throw;
}
}
private void Initialize(Surface encoderSurface, int width, int height, int rotation)
{
_rotation = rotation;
_display = EGL14.EglGetDisplay(EGL14.EglDefaultDisplay);
if (_display is null || _display.Equals(EGL14.EglNoDisplay))
throw EglError("Obtention de l'affichage EGL");
if (!EGL14.EglInitialize(_display, new int[2], 0, new int[2], 0))
throw EglError("Initialisation EGL");
int[] attributes =
[
EGL14.EglRedSize, 8, EGL14.EglGreenSize, 8, EGL14.EglBlueSize, 8,
EGL14.EglRenderableType, EGL14.EglOpenglEs2Bit,
EGL14.EglSurfaceType, EGL14.EglWindowBit, RecordableAndroid, 1, EGL14.EglNone
];
var configs = new EGLConfig[1];
var count = new int[1];
if (!EGL14.EglChooseConfig(_display, attributes, 0, configs, 0, 1, count, 0) || count[0] == 0)
throw EglError("Configuration EGL pour l'encodeur");
_context = EGL14.EglCreateContext(_display, configs[0], EGL14.EglNoContext,
[EGL14.EglContextClientVersion, 2, EGL14.EglNone], 0);
if (_context is null || _context.Equals(EGL14.EglNoContext))
throw EglError("Creation du contexte EGL");
_window = EGL14.EglCreateWindowSurface(_display, configs[0], encoderSurface, [EGL14.EglNone], 0);
if (_window is null || _window.Equals(EGL14.EglNoSurface))
throw EglError("Creation de la surface EGL");
if (!EGL14.EglMakeCurrent(_display, _window, _window, _context))
throw EglError("Activation du contexte EGL");
_program = CreateProgram();
var ids = new int[1];
GLES20.GlGenTextures(1, ids, 0);
_textureId = ids[0];
GLES20.GlBindTexture(ExternalTexture, _textureId);
GLES20.GlTexParameteri(ExternalTexture, GLES20.GlTextureMinFilter, GLES20.GlLinear);
GLES20.GlTexParameteri(ExternalTexture, GLES20.GlTextureMagFilter, GLES20.GlLinear);
GLES20.GlTexParameteri(ExternalTexture, GLES20.GlTextureWrapS, GLES20.GlClampToEdge);
GLES20.GlTexParameteri(ExternalTexture, GLES20.GlTextureWrapT, GLES20.GlClampToEdge);
_vertices = CreateBuffer([-1, -1, 1, -1, -1, 1, 1, 1]);
_coordinates = CreateBuffer(CameraFrameOrientation.GetTextureCoordinates(0));
GLES20.GlUseProgram(_program);
BindAttribute("position", _vertices);
BindAttribute("textureCoordinate", _coordinates);
_matrixLocation = GLES20.GlGetUniformLocation(_program, "textureMatrix");
GLES20.GlUniform1i(GLES20.GlGetUniformLocation(_program, "cameraTexture"), 0);
GLES20.GlViewport(0, 0, rotation is 90 or 270 ? height : width, rotation is 90 or 270 ? width : height);
CheckGl();
_texture = new SurfaceTexture(_textureId);
_texture.SetDefaultBufferSize(width, height);
_listener = new FrameListener(this);
_texture.SetOnFrameAvailableListener(_listener, _handler);
_cameraSurface = new Surface(_texture);
}
internal Task StartAsync() => InvokeAsync(() => _rendering = true);
private void DrawFrame(SurfaceTexture texture)
{
if (!_rendering) return;
try
{
texture.UpdateTexImage();
var presentationTime = _clock.GetPresentationTime(texture.Timestamp, Java.Lang.JavaSystem.NanoTime());
if (presentationTime is null) return;
texture.GetTransformMatrix(_textureMatrix);
var remainingRotation = CameraFrameOrientation.GetRemainingRotation(_rotation, _textureMatrix);
if (_remainingRotation != remainingRotation)
{
_remainingRotation = remainingRotation;
FirstTextureTransform = string.Join(", ", _textureMatrix);
_coordinates!.Position(0);
_coordinates.Put(CameraFrameOrientation.GetTextureCoordinates(remainingRotation));
_coordinates.Position(0);
global::Android.Util.Log.Info("PostIt.Podcast",
$"Camera texture transform: [{FirstTextureTransform}], remaining rotation={remainingRotation}");
}
GLES20.GlUniformMatrix4fv(_matrixLocation, 1, false, _textureMatrix, 0);
GLES20.GlDrawArrays(GLES20.GlTriangleStrip, 0, 4);
CheckGl();
if (!EGLExt.EglPresentationTimeANDROID(_display, _window, presentationTime.Value))
throw EglError("Horodatage de l'image camera");
if (!EGL14.EglSwapBuffers(_display, _window))
throw EglError("Envoi de l'image a l'encodeur");
}
catch (Exception ex)
{
_rendering = false;
_failed(new InvalidOperationException("Rotation de la capture Android impossible.", ex));
}
}
private Task InvokeAsync(Action action)
{
var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
if (!_handler.Post(() =>
{
try { action(); completion.SetResult(); }
catch (Exception ex) { completion.SetException(ex); }
}))
completion.SetException(new InvalidOperationException("Le thread de rotation camera est arrete."));
return completion.Task;
}
private static FloatBuffer CreateBuffer(float[] values)
{
using var bytes = ByteBuffer.AllocateDirect(values.Length * sizeof(float))
?? throw new InvalidOperationException("Allocation du tampon OpenGL impossible.");
bytes.Order(ByteOrder.NativeOrder()!);
var buffer = bytes.AsFloatBuffer()
?? throw new InvalidOperationException("Creation du tampon OpenGL impossible.");
buffer.Put(values);
buffer.Position(0);
return buffer;
}
private void BindAttribute(string name, FloatBuffer buffer)
{
var location = GLES20.GlGetAttribLocation(_program, name);
GLES20.GlEnableVertexAttribArray(location);
GLES20.GlVertexAttribPointer(location, 2, GLES20.GlFloat, false, 0, buffer);
}
private static int CreateShader(int type, string source)
{
var shader = GLES20.GlCreateShader(type);
GLES20.GlShaderSource(shader, source);
GLES20.GlCompileShader(shader);
var status = new int[1];
GLES20.GlGetShaderiv(shader, GLES20.GlCompileStatus, status, 0);
if (status[0] != 0) return shader;
var error = GLES20.GlGetShaderInfoLog(shader);
GLES20.GlDeleteShader(shader);
throw new InvalidOperationException($"Compilation du shader camera impossible : {error}");
}
private static int CreateProgram()
{
var vertex = CreateShader(GLES20.GlVertexShader, VertexShader);
var fragment = 0;
var program = 0;
try
{
fragment = CreateShader(GLES20.GlFragmentShader, FragmentShader);
program = GLES20.GlCreateProgram();
GLES20.GlAttachShader(program, vertex);
GLES20.GlAttachShader(program, fragment);
GLES20.GlLinkProgram(program);
var status = new int[1];
GLES20.GlGetProgramiv(program, GLES20.GlLinkStatus, status, 0);
if (status[0] == 0)
throw new InvalidOperationException($"Edition du programme camera impossible : {GLES20.GlGetProgramInfoLog(program)}");
return program;
}
catch
{
if (program != 0) GLES20.GlDeleteProgram(program);
throw;
}
finally
{
GLES20.GlDeleteShader(vertex);
if (fragment != 0) GLES20.GlDeleteShader(fragment);
}
}
private static void CheckGl()
{
var error = GLES20.GlGetError();
if (error != GLES20.GlNoError)
throw new InvalidOperationException($"Erreur OpenGL de capture : 0x{error:X}.");
}
private static InvalidOperationException EglError(string operation)
=> new($"{operation} impossible : EGL 0x{EGL14.EglGetError():X}.");
public void Dispose()
{
if (_disposed) return;
_disposed = true;
try { InvokeAsync(Release).GetAwaiter().GetResult(); }
finally
{
_thread.QuitSafely();
_thread.Join();
_handler.Dispose();
_thread.Dispose();
}
}
private void Release()
{
_rendering = false;
_texture?.SetOnFrameAvailableListener(null);
_cameraSurface?.Release();
_cameraSurface?.Dispose();
_texture?.Release();
_texture?.Dispose();
_listener?.Dispose();
if (_program != 0) GLES20.GlDeleteProgram(_program);
if (_textureId != 0) GLES20.GlDeleteTextures(1, [_textureId], 0);
_vertices?.Dispose();
_coordinates?.Dispose();
if (_display is not null && !_display.Equals(EGL14.EglNoDisplay))
{
EGL14.EglMakeCurrent(_display, EGL14.EglNoSurface, EGL14.EglNoSurface, EGL14.EglNoContext);
if (_window is not null && !_window.Equals(EGL14.EglNoSurface))
EGL14.EglDestroySurface(_display, _window);
if (_context is not null && !_context.Equals(EGL14.EglNoContext))
EGL14.EglDestroyContext(_display, _context);
EGL14.EglReleaseThread();
EGL14.EglTerminate(_display);
}
_window?.Dispose();
_context?.Dispose();
_display?.Dispose();
}
private sealed class FrameListener(AndroidCameraFrameRotator owner)
: Java.Lang.Object, SurfaceTexture.IOnFrameAvailableListener
{
public void OnFrameAvailable(SurfaceTexture? texture)
{
if (texture is not null) owner.DrawFrame(texture);
}
}
}

View file

@ -26,6 +26,7 @@ internal sealed class AndroidPodcastCapture : IPodcastCapture
private CameraDevice? _camera;
private CameraCaptureSession? _session;
private MediaRecorder? _recorder;
private AndroidCameraFrameRotator? _rotator;
private Surface? _surface;
private ParcelFileDescriptor? _readPipe;
private ParcelFileDescriptor? _writePipe;
@ -36,6 +37,8 @@ internal sealed class AndroidPodcastCapture : IPodcastCapture
private bool _started;
private bool _disposed;
public Task Completion => _completion.Task;
internal string OrientationDiagnostics { get; private set; } = "";
internal string? TextureTransform => _rotator?.FirstTextureTransform;
private AndroidPodcastCapture(PodcastCaptureService service) => _service = service;
@ -103,6 +106,30 @@ internal sealed class AndroidPodcastCapture : IPodcastCapture
var size = sizes.Where(s => s.Width <= 640 && s.Height <= 480)
.OrderByDescending(s => s.Width * s.Height).FirstOrDefault()
?? throw new InvalidOperationException("La camera ne propose aucun format jusqu'a 480p.");
using var sensor = characteristics.Get(CameraCharacteristics.SensorOrientation) as Java.Lang.Integer
?? throw new InvalidOperationException("Orientation du capteur camera indisponible.");
using var facing = characteristics.Get(CameraCharacteristics.LensFacing) as Java.Lang.Integer
?? throw new InvalidOperationException("Position du capteur camera indisponible.");
var activity = MainActivity.Current ?? throw new InvalidOperationException("Activite Android indisponible.");
#pragma warning disable CS0618, CA1422
var display = OperatingSystem.IsAndroidVersionAtLeast(30)
? activity.Display : activity.WindowManager?.DefaultDisplay;
#pragma warning restore CS0618, CA1422
var displayRotation = display?.Rotation
?? throw new InvalidOperationException("Orientation de l'ecran Android indisponible.");
var displayDegrees = displayRotation switch
{
SurfaceOrientation.Rotation0 => 0,
SurfaceOrientation.Rotation90 => 90,
SurfaceOrientation.Rotation180 => 180,
SurfaceOrientation.Rotation270 => 270,
_ => throw new InvalidOperationException($"Rotation d'ecran Android inconnue : {displayRotation}.")
};
var rotation = CameraFrameOrientation.GetRotation(sensor.IntValue(), displayDegrees,
facing.IntValue() == (int)LensFacing.Front);
global::Android.Util.Log.Info("PostIt.Podcast",
$"Camera orientation: sensor={sensor.IntValue()}, display={displayDegrees}, facing={facing.IntValue()}, rotation={rotation}");
OrientationDiagnostics = $"sensor={sensor.IntValue()}, display={displayDegrees}, facing={facing.IntValue()}, rotation={rotation}";
#pragma warning disable CS0618
_recorder = OperatingSystem.IsAndroidVersionAtLeast(31)
? new MediaRecorder(_service) : new MediaRecorder();
@ -111,7 +138,8 @@ internal sealed class AndroidPodcastCapture : IPodcastCapture
if (microphone) _recorder.SetAudioSource(AudioSource.Mic);
_recorder.SetVideoSource(VideoSource.Surface);
_recorder.SetOutputFormat(OutputFormat.Mpeg2Ts);
_recorder.SetVideoSize(size.Width, size.Height);
_recorder.SetVideoSize(rotation is 90 or 270 ? size.Height : size.Width,
rotation is 90 or 270 ? size.Width : size.Height);
_recorder.SetVideoFrameRate(30);
_recorder.SetVideoEncodingBitRate(1200000);
_recorder.SetVideoEncoder(VideoEncoder.H264);
@ -131,20 +159,23 @@ internal sealed class AndroidPodcastCapture : IPodcastCapture
ct.ThrowIfCancellationRequested();
if (_stopRequested) throw new System.OperationCanceledException("Demarrage de la capture annule.");
_surface = _recorder.Surface ?? throw new InvalidOperationException("Surface encodeur indisponible.");
_rotator = await AndroidCameraFrameRotator.CreateAsync(_surface, size.Width, size.Height, rotation, Fail);
ct.ThrowIfCancellationRequested();
if (_stopRequested) throw new System.OperationCanceledException("Demarrage de la capture annule.");
var cameraSurface = _rotator.CameraSurface;
var opened = new TaskCompletionSource<CameraDevice>(TaskCreationOptions.RunContinuationsAsynchronously);
manager.OpenCamera(cameraId, new DeviceCallback(this, opened), _handler);
_camera = await opened.Task.WaitAsync(TimeSpan.FromSeconds(10), ct);
var configured = new TaskCompletionSource<CameraCaptureSession>(TaskCreationOptions.RunContinuationsAsynchronously);
#pragma warning disable CS0618, CA1422
_camera.CreateCaptureSession(new List<Surface> { _surface }, new SessionCallback(this, configured), _handler);
_camera.CreateCaptureSession(new List<Surface> { cameraSurface }, new SessionCallback(this, configured), _handler);
#pragma warning restore CS0618, CA1422
_session = await configured.Task.WaitAsync(TimeSpan.FromSeconds(10), ct);
ct.ThrowIfCancellationRequested();
if (_stopRequested) throw new System.OperationCanceledException("Demarrage de la capture annule.");
using var request = _camera.CreateCaptureRequest(CameraTemplate.Record);
request.AddTarget(_surface);
request.AddTarget(cameraSurface);
using var built = request.Build();
_session.SetRepeatingRequest(built, null, _handler);
var handle = new SafeFileHandle((IntPtr)_readPipe.Fd, ownsHandle: false);
var stream = new FileStream(handle, FileAccess.Read);
_pump = Task.Run(async () =>
@ -169,6 +200,8 @@ internal sealed class AndroidPodcastCapture : IPodcastCapture
});
await Task.Run(_recorder.Start, ct);
_started = true;
await _rotator.StartAsync();
_session.SetRepeatingRequest(built, null, _handler);
_writePipe.Close();
_writePipe.Dispose();
_writePipe = null;
@ -261,7 +294,11 @@ internal sealed class AndroidPodcastCapture : IPodcastCapture
private void StopNative()
{
try { if (_started) _recorder?.Stop(); }
try
{
// Keep frames flowing until MediaRecorder has drained its encoder.
if (_started) _recorder?.Stop();
}
catch (Java.Lang.RuntimeException ex)
{
_failure = ex;
@ -276,6 +313,8 @@ internal sealed class AndroidPodcastCapture : IPodcastCapture
_camera?.Close();
_camera?.Dispose();
_camera = null;
_rotator?.Dispose();
_rotator = null;
if (_recorder is not null)
{
_recorder.Error -= OnRecorderError;

View file

@ -0,0 +1,24 @@
using System;
namespace PostIt.Android.Services;
internal sealed class CameraFrameClock
{
private long? _offset;
private long? _previousTimestamp;
internal long? GetPresentationTime(long cameraTimestamp, long monotonicNow)
{
if (_previousTimestamp is { } previous)
{
if (cameraTimestamp == previous) return null;
if (cameraTimestamp < previous)
throw new InvalidOperationException("Les horodatages de la camera Android ont recule.");
}
// Camera2 may use elapsed realtime (including suspend), whereas MediaRecorder
// and its microphone use monotonic time. Preserve frame intervals, not that epoch.
_offset ??= monotonicNow - cameraTimestamp;
_previousTimestamp = cameraTimestamp;
return checked(cameraTimestamp + _offset.Value);
}
}

View file

@ -0,0 +1,55 @@
using System;
namespace PostIt.Android.Services;
internal static class CameraFrameOrientation
{
internal static int GetRotation(int sensorDegrees, int displayDegrees, bool frontFacing)
{
Validate(sensorDegrees);
Validate(displayDegrees);
return (sensorDegrees + (frontFacing ? displayDegrees : -displayDegrees) + 360) % 360;
}
internal static float[] GetTextureCoordinates(int clockwiseDegrees)
{
Validate(clockwiseDegrees);
float[] corners = [0, 0, 1, 0, 0, 1, 1, 1];
for (var i = 0; i < corners.Length; i += 2)
{
var x = corners[i];
var y = corners[i + 1];
(corners[i], corners[i + 1]) = clockwiseDegrees switch
{
90 => (1 - y, x),
180 => (1 - x, 1 - y),
270 => (y, 1 - x),
_ => (x, y)
};
}
return corners;
}
internal static int GetRemainingRotation(int clockwiseDegrees, float[] textureMatrix)
{
Validate(clockwiseDegrees);
if (textureMatrix.Length != 16)
throw new ArgumentException("A SurfaceTexture transform must contain 16 elements.", nameof(textureMatrix));
// SurfaceTexture's usual Y flip changes the sign of the first column's Y.
// Camera2 can also put the sensor rotation in this matrix; don't apply it twice.
var angle = Math.Atan2(-textureMatrix[1], textureMatrix[0]) * 180 / Math.PI;
if (!double.IsFinite(angle) || (textureMatrix[0] == 0 && textureMatrix[1] == 0))
throw new ArgumentException("Invalid SurfaceTexture orientation.", nameof(textureMatrix));
var quarterTurns = (int)Math.Round(angle / 90);
if (Math.Abs(angle - quarterTurns * 90) > 0.01)
throw new ArgumentException("SurfaceTexture orientation is not a quarter turn.", nameof(textureMatrix));
var producerRotation = (quarterTurns * 90 + 360) % 360;
return (clockwiseDegrees - producerRotation + 360) % 360;
}
private static void Validate(int degrees)
{
if (degrees is not (0 or 90 or 180 or 270))
throw new ArgumentOutOfRangeException(nameof(degrees), degrees, "Rotation must be 0, 90, 180 or 270 degrees.");
}
}

View file

@ -0,0 +1,9 @@
# Keep the LibVLC Android classes and JNI/native bindings intact in Release builds.
# R8/ProGuard otherwise can strip or rename the Java-side entry points that load
# the native libvlc and libc++_shared libraries bundled by VideoLAN.LibVLC.Android.
-keep class org.videolan.libvlc.** { *; }
-keep class org.videolan.libvlc.** { native *; }
-keep class org.videolan.medialibrary.** { *; }
-keepattributes *Annotation*
-dontwarn org.videolan.libvlc.**
-dontwarn org.videolan.medialibrary.**