[camera_android_camerax] Support NV21

packages/camera/camera_android_camerax/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.6.21
+
+* Implements NV21 support for image streaming.
+
 ## 0.6.20+3
 
 * Bumps com.google.guava:guava from 33.4.0-android to 33.4.8-android.

packages/camera/camera_android_camerax/README.md

@@ -73,7 +73,9 @@ in the merged Android manifest of your app, then take the following steps to rem
     tools:node="remove" />
 ```
 
-### Allowing image streaming in the background
+### Notes on image streaming
+
+#### Allowing image streaming in the background
 
 As of Android 14, to allow for background image streaming, you will need to specify the foreground
 [`TYPE_CAMERA`][12] foreground service permission in your app's manifest. Specifically, in
@@ -86,6 +88,12 @@ As of Android 14, to allow for background image streaming, you will need to spec
 </manifest>
 ```
 
+#### Configuring NV21 image format
+
+If you initialize a `CameraController` with `ImageFormatGroup.nv21`, then streamed images will
+still have the `ImageFormatGroup.yuv420` format, but their image data will be formatted in NV21.
+See https://developer.android.com/reference/kotlin/androidx/camera/core/ImageAnalysis#OUTPUT_IMAGE_FORMAT_NV21().
+
 ## Contributing
 
 For more information on contributing to this plugin, see [`CONTRIBUTING.md`](CONTRIBUTING.md).

packages/camera/camera_android_camerax/android/build.gradle

@@ -40,8 +40,8 @@ android {
     }
 
     defaultConfig {
-        // Many of the CameraX APIs require API 21.
-        minSdkVersion 21
+        // CameraX APIs require API 23 or later.
+        minSdk = 23
         testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
     }
 
@@ -72,7 +72,7 @@ android {
 
 dependencies {
     // CameraX core library using the camera2 implementation must use same version number.
-    def camerax_version = "1.5.0-beta01"
+    def camerax_version = "1.5.0-rc01"
     implementation "androidx.camera:camera-core:${camerax_version}"
     implementation "androidx.camera:camera-camera2:${camerax_version}"
     implementation "androidx.camera:camera-lifecycle:${camerax_version}"

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/CameraXLibrary.g.kt

@@ -1,7 +1,7 @@
 // Copyright 2013 The Flutter Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
-// Autogenerated from Pigeon (v25.3.2), do not edit directly.
+// Autogenerated from Pigeon (v25.5.0), do not edit directly.
 // See also: https://pub.dev/packages/pigeon
 @file:Suppress("UNCHECKED_CAST", "ArrayInDataClass")
 
@@ -4761,7 +4761,8 @@ abstract class PigeonApiImageAnalysis(
 ) {
   abstract fun pigeon_defaultConstructor(
       resolutionSelector: androidx.camera.core.resolutionselector.ResolutionSelector?,
-      targetRotation: Long?
+      targetRotation: Long?,
+      outputImageFormat: Long?
   ): androidx.camera.core.ImageAnalysis
 
   abstract fun resolutionSelector(
@@ -4800,10 +4801,12 @@ abstract class PigeonApiImageAnalysis(
             val resolutionSelectorArg =
                 args[1] as androidx.camera.core.resolutionselector.ResolutionSelector?
             val targetRotationArg = args[2] as Long?
+            val outputImageFormatArg = args[3] as Long?
             val wrapped: List<Any?> =
                 try {
                   api.pigeonRegistrar.instanceManager.addDartCreatedInstance(
-                      api.pigeon_defaultConstructor(resolutionSelectorArg, targetRotationArg),
+                      api.pigeon_defaultConstructor(
+                          resolutionSelectorArg, targetRotationArg, outputImageFormatArg),
                       pigeon_identifierArg)
                   listOf(null)
                 } catch (exception: Throwable) {

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ImageAnalysisProxyApi.java

@@ -21,14 +21,21 @@ class ImageAnalysisProxyApi extends PigeonApiImageAnalysis {
   @NonNull
   @Override
   public ImageAnalysis pigeon_defaultConstructor(
-      @Nullable ResolutionSelector resolutionSelector, @Nullable Long targetRotation) {
+      @Nullable ResolutionSelector resolutionSelector,
+      @Nullable Long targetRotation,
+      @Nullable Long outputImageFormat) {
     final ImageAnalysis.Builder builder = new ImageAnalysis.Builder();
     if (resolutionSelector != null) {
       builder.setResolutionSelector(resolutionSelector);
     }
     if (targetRotation != null) {
       builder.setTargetRotation(targetRotation.intValue());
     }
+
+    if (outputImageFormat != null) {
+      builder.setOutputImageFormat(outputImageFormat.intValue());
+    }
+
     return builder.build();
   }
 

packages/camera/camera_android_camerax/android/src/test/java/io/flutter/plugins/camerax/Camera2CameraInfoTest.java

@@ -69,7 +69,7 @@ public void getCameraCharacteristic_returnsCorrespondingValueOfKeyWhenKeyNotReco
     assertEquals(value, api.getCameraCharacteristic(instance, key));
   }
 
-  @Config(minSdk = 21)
+  @Config(minSdk = 23)
   @SuppressWarnings("unchecked")
   @Test
   public void getCameraCharacteristic_returnsExpectedCameraHardwareLevelWhenRequested() {

packages/camera/camera_android_camerax/android/src/test/java/io/flutter/plugins/camerax/ImageAnalysisTest.java

@@ -32,11 +32,13 @@ public void pigeon_defaultConstructor_createsExpectedImageAnalysisInstance() {
 
     final ResolutionSelector mockResolutionSelector = new ResolutionSelector.Builder().build();
     final long targetResolution = Surface.ROTATION_0;
+    final long outputImageFormat = ImageAnalysis.OUTPUT_IMAGE_FORMAT_NV21;
     final ImageAnalysis imageAnalysis =
-        api.pigeon_defaultConstructor(mockResolutionSelector, targetResolution);
+        api.pigeon_defaultConstructor(mockResolutionSelector, targetResolution, outputImageFormat);
 
     assertEquals(imageAnalysis.getResolutionSelector(), mockResolutionSelector);
     assertEquals(imageAnalysis.getTargetRotation(), Surface.ROTATION_0);
+    assertEquals(imageAnalysis.getOutputImageFormat(), ImageAnalysis.OUTPUT_IMAGE_FORMAT_NV21);
   }
 
   @Test

packages/camera/camera_android_camerax/android/src/test/java/io/flutter/plugins/camerax/PendingRecordingTest.java

@@ -98,7 +98,7 @@ public void start_callsStartOnInstance() {
           .when(() -> ContextCompat.getMainExecutor(any()))
           .thenAnswer((Answer<Executor>) invocation -> mock(Executor.class));
 
-      when(instance.start(any(), any())).thenReturn(value);
+      when(instance.start(any(Executor.class), any())).thenReturn(value);
 
       assertEquals(value, api.start(instance, listener));
     }

packages/camera/camera_android_camerax/example/lib/camera_image.dart

@@ -83,6 +83,9 @@ ImageFormatGroup _asImageFormatGroup(dynamic rawFormat) {
       // android.graphics.ImageFormat.YUV_420_888
       case 35:
         return ImageFormatGroup.yuv420;
+      // android.graphics.ImageFormat.NV21
+      case 17:
+        return ImageFormatGroup.nv21;
       // android.graphics.ImageFormat.JPEG
       case 256:
         return ImageFormatGroup.jpeg;

packages/camera/camera_android_camerax/lib/src/android_camera_camerax.dart

@@ -174,15 +174,30 @@ class AndroidCameraCameraX extends CameraPlatform {
   @visibleForTesting
   StreamController<CameraImageData>? cameraImageDataStreamController;
 
-  /// Constant representing the multi-plane Android YUV 420 image format.
+  /// Constant representing the multi-plane Android YUV 420 image format used by ImageProxy.
   ///
   /// See https://developer.android.com/reference/android/graphics/ImageFormat#YUV_420_888.
-  static const int imageFormatYuv420_888 = 35;
+  static const int imageProxyFormatYuv420_888 = 35;
 
-  /// Constant representing the compressed JPEG image format.
+  /// Constant representing the NV21 image format used by ImageProxy.
+  ///
+  /// See https://developer.android.com/reference/android/graphics/ImageFormat#NV21.
+  static const int imageProxyFormatNv21 = 17;
+
+  /// Constant representing the compressed JPEG image format used by ImageProxy.
   ///
   /// See https://developer.android.com/reference/android/graphics/ImageFormat#JPEG.
-  static const int imageFormatJpeg = 256;
+  static const int imageProxyFormatJpeg = 256;
+
+  /// Constant representing the YUV 420 image format used for configuring ImageAnalysis.
+  ///
+  /// See https://developer.android.com/reference/androidx/camera/core/ImageAnalysis#OUTPUT_IMAGE_FORMAT_YUV_420_888()
+  static const int imageAnalysisOutputImageFormatYuv420_888 = 1;
+
+  /// Constant representing the NV21 image format used for configuring ImageAnalysis.
+  ///
+  /// See https://developer.android.com/reference/androidx/camera/core/ImageAnalysis#OUTPUT_IMAGE_FORMAT_NV21().
+  static const int imageAnalysisOutputImageFormatNv21 = 3;
 
   /// Error code indicating a [ZoomState] was requested, but one has not been
   /// set for the camera in use.
@@ -269,6 +284,12 @@ class AndroidCameraCameraX extends CameraPlatform {
   /// A map to associate a [CameraInfo] with its camera name.
   final Map<String, CameraInfo> _savedCameras = <String, CameraInfo>{};
 
+  /// The preset resolution selector for the camera.
+  ResolutionSelector? _presetResolutionSelector;
+
+  /// The ID of the surface texture that the camera preview is drawn to.
+  late int _flutterSurfaceTextureId;
+
   /// Returns list of all available cameras and their descriptions.
   @override
   Future<List<CameraDescription>> availableCameras() async {
@@ -380,8 +401,9 @@ class AndroidCameraCameraX extends CameraPlatform {
     );
     // Determine ResolutionSelector and QualitySelector based on
     // resolutionPreset for camera UseCases.
-    final ResolutionSelector? presetResolutionSelector =
-        _getResolutionSelectorFromPreset(mediaSettings?.resolutionPreset);
+    _presetResolutionSelector = _getResolutionSelectorFromPreset(
+      mediaSettings?.resolutionPreset,
+    );
     final QualitySelector? presetQualitySelector =
         _getQualitySelectorFromPreset(mediaSettings?.resolutionPreset);
 
@@ -391,55 +413,26 @@ class AndroidCameraCameraX extends CameraPlatform {
 
     // Configure Preview instance.
     preview = proxy.newPreview(
-      resolutionSelector: presetResolutionSelector,
+      resolutionSelector: _presetResolutionSelector,
       /* use CameraX default target rotation */ targetRotation: null,
     );
-    final int flutterSurfaceTextureId = await preview!.setSurfaceProvider(
+    _flutterSurfaceTextureId = await preview!.setSurfaceProvider(
       systemServicesManager,
     );
 
     // Configure ImageCapture instance.
     imageCapture = proxy.newImageCapture(
-      resolutionSelector: presetResolutionSelector,
+      resolutionSelector: _presetResolutionSelector,
       /* use CameraX default target rotation */ targetRotation:
           await deviceOrientationManager.getDefaultDisplayRotation(),
     );
 
-    // Configure ImageAnalysis instance.
-    // Defaults to YUV_420_888 image format.
-    imageAnalysis = proxy.newImageAnalysis(
-      resolutionSelector: presetResolutionSelector,
-      /* use CameraX default target rotation */ targetRotation: null,
-    );
-
     // Configure VideoCapture and Recorder instances.
     recorder = proxy.newRecorder(qualitySelector: presetQualitySelector);
     videoCapture = proxy.withOutputVideoCapture(videoOutput: recorder!);
 
-    // Bind configured UseCases to ProcessCameraProvider instance & mark Preview
-    // instance as bound but not paused. Video capture is bound at first use
-    // instead of here.
-    camera = await processCameraProvider!.bindToLifecycle(
-      cameraSelector!,
-      <UseCase>[preview!, imageCapture!, imageAnalysis!],
-    );
-    await _updateCameraInfoAndLiveCameraState(flutterSurfaceTextureId);
-    previewInitiallyBound = true;
-    _previewIsPaused = false;
-
     // Retrieve info required for correcting the rotation of the camera preview
     // if necessary.
-
-    final Camera2CameraInfo camera2CameraInfo = proxy.fromCamera2CameraInfo(
-      cameraInfo: cameraInfo!,
-    );
-    sensorOrientationDegrees =
-        ((await camera2CameraInfo.getCameraCharacteristic(
-                  proxy.sensorOrientationCameraCharacteristics(),
-                ))!
-                as int)
-            .toDouble();
-
     sensorOrientationDegrees = cameraDescription.sensorOrientation.toDouble();
     _handlesCropAndRotation = await preview!
         .surfaceProducerHandlesCropAndRotation();
@@ -449,35 +442,59 @@ class AndroidCameraCameraX extends CameraPlatform {
     _initialDefaultDisplayRotation = await deviceOrientationManager
         .getDefaultDisplayRotation();
 
-    return flutterSurfaceTextureId;
+    return _flutterSurfaceTextureId;
   }
 
   /// Initializes the camera on the device.
   ///
-  /// Since initialization of a camera does not directly map as an operation to
-  /// the CameraX library, this method just retrieves information about the
-  /// camera and sends a [CameraInitializedEvent].
+  /// Specifically, this method:
+  ///  * Configures the [ImageAnalysis] instance according to the specified
+  ///   [imageFormatGroup]
+  ///  * Binds the configured [Preview], [ImageCapture], and [ImageAnalysis]
+  ///    instances to the [ProcessCameraProvider] instance.
+  ///  * Retrieves information about the camera and sends a [CameraInitializedEvent].
   ///
   /// [imageFormatGroup] is used to specify the image format used for image
-  /// streaming, but CameraX currently only supports YUV_420_888 (supported by
-  /// Flutter) and RGBA (not supported by Flutter). CameraX uses YUV_420_888
-  /// by default, so [imageFormatGroup] is not used.
+  /// streaming, but CameraX currently only supports YUV_420_888 (the CameraX default),
+  /// NV21, and RGBA (not supported by Flutter).
   @override
   Future<void> initializeCamera(
     int cameraId, {
     ImageFormatGroup imageFormatGroup = ImageFormatGroup.unknown,
   }) async {
-    // Configure CameraInitializedEvent to send as representation of a
-    // configured camera:
-    // Retrieve preview resolution.
+    // If preview has not been created, then no camera has been created, which signals that
+    // createCamera was not called before initializeCamera.
     if (preview == null) {
-      // No camera has been created; createCamera must be called before initializeCamera.
       throw CameraException(
         'cameraNotFound',
         "Camera not found. Please call the 'create' method before calling 'initialize'",
       );
     }
+    // Configure ImageAnalysis instance.
+    // Defaults to YUV_420_888 image format.
+    imageAnalysis = proxy.newImageAnalysis(
+      resolutionSelector: _presetResolutionSelector,
+      outputImageFormat: _imageAnalysisOutputFormatFromImageFormatGroup(
+        imageFormatGroup,
+      ),
+      /* use CameraX default target rotation */ targetRotation: null,
+    );
+
+    // Bind configured UseCases to ProcessCameraProvider instance & mark Preview
+    // instance as bound but not paused. Video capture is bound at first use
+    // instead of here.
+    camera = await processCameraProvider!.bindToLifecycle(
+      cameraSelector!,
+      <UseCase>[preview!, imageCapture!, imageAnalysis!],
+    );
+    await _updateCameraInfoAndLiveCameraState(_flutterSurfaceTextureId);
+    previewInitiallyBound = true;
+    _previewIsPaused = false;
+
+    // Configure CameraInitializedEvent to send as representation of a
+    // configured camera:
 
+    // Retrieve preview resolution.
     final ResolutionInfo previewResolutionInfo = (await preview!
         .getResolutionInfo())!;
 
@@ -1214,6 +1231,10 @@ class AndroidCameraCameraX extends CameraPlatform {
   /// implementation using a broadcast [StreamController], which does not
   /// support those operations.
   ///
+  /// If the camera was initialized with [ImageFormatGroup.nv21], then the
+  /// streamed images will still have format [ImageFormatGroup.yuv420], but
+  /// their image data will be formatted in NV21.
+  ///
   /// [cameraId] and [options] are not used.
   @override
   Stream<CameraImageData> onStreamedFrameAvailable(
@@ -1326,14 +1347,30 @@ class AndroidCameraCameraX extends CameraPlatform {
     await imageAnalysis!.clearAnalyzer();
   }
 
-  /// Converts between Android ImageFormat constants and [ImageFormatGroup]s.
+  /// Converts [ImageFormatGroup]s to Android ImageAnalysis output format constants.
+  ///
+  /// See https://developer.android.com/reference/androidx/camera/core/ImageAnalysis.
+  int? _imageAnalysisOutputFormatFromImageFormatGroup(dynamic format) {
+    switch (format) {
+      case ImageFormatGroup.yuv420:
+        return imageAnalysisOutputImageFormatYuv420_888;
+      case ImageFormatGroup.nv21:
+        return imageAnalysisOutputImageFormatNv21;
+    }
+
+    return null;
+  }
+
+  /// Converts from Android ImageFormat constants to [ImageFormatGroup]s.
   ///
   /// See https://developer.android.com/reference/android/graphics/ImageFormat.
   ImageFormatGroup _imageFormatGroupFromPlatformData(dynamic data) {
     switch (data) {
-      case imageFormatYuv420_888: // android.graphics.ImageFormat.YUV_420_888
+      case imageProxyFormatYuv420_888: // android.graphics.ImageFormat.YUV_420_888
         return ImageFormatGroup.yuv420;
-      case imageFormatJpeg: // android.graphics.ImageFormat.JPEG
+      case imageProxyFormatNv21: // android.graphics.ImageFormat.NV21
+        return ImageFormatGroup.nv21;
+      case imageProxyFormatJpeg: // android.graphics.ImageFormat.JPEG
         return ImageFormatGroup.jpeg;
     }