MN-469 Fixing portrait camera + prevent zoom/wide camera (#676)

* fixing camera portrait and prevent zoom/wide camera

* Fixed PR / deviceId and facingMode removed from camera props

Author: dlymonk

packages/public/camera-web/src/Camera/Camera.tsx

@@ -19,7 +19,7 @@ import { CameraEventHandlers, CameraHUDComponent } from './CameraHUD.types';
  * Props given to the Camera component.
  */
 export interface CameraProps
-  extends Partial<CameraConfig>,
+  extends Partial<Pick<CameraConfig, 'resolution'>>,
     Partial<CompressionOptions>,
     CameraEventHandlers {
   /**
@@ -43,9 +43,7 @@ export interface CameraProps
  * component works.
  */
 export function Camera({
-  facingMode = CameraFacingMode.ENVIRONMENT,
   resolution = CameraResolution.UHD_4K,
-  deviceId,
   format = CompressionFormat.JPEG,
   quality = 0.8,
   HUDComponent,
@@ -58,7 +56,7 @@ export function Camera({
     error,
     retry,
     isLoading: isPreviewLoading,
-  } = useCameraPreview({ facingMode, resolution, deviceId });
+  } = useCameraPreview({ resolution, facingMode: CameraFacingMode.ENVIRONMENT });
   const { ref: canvasRef } = useCameraCanvas({ dimensions });
   const { takeScreenshot } = useCameraScreenshot({ videoRef, canvasRef, dimensions });
   const { compress } = useCompression({ canvasRef, options: { format, quality } });
@@ -76,7 +74,7 @@ export function Camera({
           style={styles['cameraPreview']}
           ref={videoRef}
           autoPlay
-          playsInline
+          playsInline={true}
           controls={false}
           data-testid='camera-video-preview'
         />

packages/public/camera-web/src/Camera/hooks/useUserMedia.ts

@@ -2,6 +2,8 @@ import { useMonitoring } from '@monkvision/monitoring';
 import deepEqual from 'fast-deep-equal';
 import { useEffect, useState } from 'react';
 import { PixelDimensions } from '@monkvision/types';
+import { isMobileDevice } from '@monkvision/common';
+import { getValidCameraDeviceIds } from './utils';
 
 /**
  * Enumeration of the different Native error names that can happen when a stream is invalid.
@@ -133,6 +135,13 @@ function getStreamDimensions(stream: MediaStream): PixelDimensions {
   return { width, height };
 }
 
+function swapWidthAndHeight(dimensions: PixelDimensions): PixelDimensions {
+  return {
+    width: dimensions.height,
+    height: dimensions.width,
+  };
+}
+
 /**
  * React hook that wraps the `navigator.mediaDevices.getUserMedia` browser function in order to add React logic layers
  * and utility tools :
@@ -202,10 +211,20 @@ export function useUserMedia(constraints: MediaStreamConstraints): UserMediaResu
           stream.removeEventListener('inactive', onStreamInactive);
           stream.getTracks().forEach((track) => track.stop());
         }
-        const str = await navigator.mediaDevices.getUserMedia(constraints);
+        const cameraDeviceIds = await getValidCameraDeviceIds(constraints);
+        const updatedConstraints = {
+          ...constraints,
+          video: {
+            ...(constraints ? (constraints.video as MediaStreamConstraints) : null),
+            deviceId: { exact: cameraDeviceIds },
+          },
+        };
+        const str = await navigator.mediaDevices.getUserMedia(updatedConstraints);
         str?.addEventListener('inactive', onStreamInactive);
         setStream(str);
-        setDimensions(getStreamDimensions(str));
+
+        const dimensionsStr = getStreamDimensions(str);
+        setDimensions(isMobileDevice() ? swapWidthAndHeight(dimensionsStr) : dimensionsStr);
         setIsLoading(false);
       } catch (err) {
         handleGetUserMediaError(err);
@@ -215,5 +234,21 @@ export function useUserMedia(constraints: MediaStreamConstraints): UserMediaResu
     getUserMedia().catch((err) => handleError(err));
   }, [constraints, stream, error, isLoading, lastConstraintsApplied, onStreamInactive]);
 
+  useEffect(() => {
+    const portrait = window.matchMedia('(orientation: portrait)');
+
+    const handleOrientationChange = () => {
+      if (stream) {
+        const dimensionsStr = getStreamDimensions(stream);
+        setDimensions(isMobileDevice() ? swapWidthAndHeight(dimensionsStr) : dimensionsStr);
+      }
+    };
+    portrait.addEventListener('change', handleOrientationChange);
+
+    return () => {
+      portrait.removeEventListener('change', handleOrientationChange);
+    };
+  }, [stream]);
+
   return { stream, dimensions, error, retry, isLoading };
 }

packages/public/camera-web/src/Camera/hooks/utils/getMediaContraints.ts

@@ -82,12 +82,6 @@ export interface CameraConfig {
    * @default `CameraResolution.UHD_4K`
    */
   resolution: CameraResolution;
-  /**
-   * The ID of the camera device to use. This ID can be fetched using the native
-   * `navigator.mediaDevices?.enumerateDevices` function. If this ID is specified, it will prevail over the `facingMode`
-   * property.
-   */
-  deviceId?: string;
 }
 
 /**
@@ -108,10 +102,6 @@ export function getMediaConstraints(config: CameraConfig): MediaStreamConstraint
     facingMode: config.facingMode,
   };
 
-  if (config.deviceId) {
-    video.deviceId = config.deviceId;
-  }
-
   return {
     audio: false,
     video,

packages/public/camera-web/src/Camera/hooks/utils/getValidCameraDeviceIds.ts

@@ -0,0 +1,25 @@
+function isValidCamera(device: MediaDeviceInfo) {
+  return (
+    device.kind === 'videoinput' &&
+    !device.label.includes('Wide') &&
+    !device.label.includes('Telephoto') &&
+    !device.label.includes('Triple') &&
+    !device.label.includes('Dual') &&
+    !device.label.includes('Ultra')
+  );
+}
+/**
+ * Retrieves the valid camera device IDs based on the specified constraints.
+ */
+export async function getValidCameraDeviceIds(
+  constraints: MediaStreamConstraints,
+): Promise<string[]> {
+  const str = await navigator.mediaDevices.getUserMedia(constraints);
+  const devices = await navigator.mediaDevices.enumerateDevices();
+  const validCameraDeviceIds = devices
+    .filter((device) => isValidCamera(device))
+    .map((device) => device.deviceId);
+
+  str.getTracks().forEach((track) => track.stop());
+  return validCameraDeviceIds;
+}

packages/public/camera-web/src/Camera/hooks/utils/index.ts

@@ -1,2 +1,3 @@
 export * from './getCanvasHandle';
 export * from './getMediaContraints';
+export * from './getValidCameraDeviceIds';

packages/public/camera-web/test/Camera/Camera.test.tsx

@@ -44,15 +44,12 @@ describe('Camera component', () => {
     jest.clearAllMocks();
   });
 
-  it('should pass the facingMode, resolution and deviceId props to the useCameraPreview hook', () => {
-    const facingMode = CameraFacingMode.USER;
+  it('should pass the resolution props to the useCameraPreview hook', () => {
+    const facingMode = CameraFacingMode.ENVIRONMENT;
     const resolution = CameraResolution.HD_720P;
-    const deviceId = 'test-device-id';
-    const { unmount } = render(
-      <Camera facingMode={facingMode} resolution={resolution} deviceId={deviceId} />,
-    );
+    const { unmount } = render(<Camera resolution={resolution} />);
 
-    expect(useCameraPreview).toHaveBeenCalledWith({ facingMode, resolution, deviceId });
+    expect(useCameraPreview).toHaveBeenCalledWith({ facingMode, resolution });
     unmount();
   });
 
@@ -78,17 +75,6 @@ describe('Camera component', () => {
     unmount();
   });
 
-  it('should not use any deviceId if not provided', () => {
-    const { unmount } = render(<Camera />);
-
-    expect(useCameraPreview).toHaveBeenCalledWith(
-      expect.objectContaining({
-        deviceId: undefined,
-      }),
-    );
-    unmount();
-  });
-
   it('should pass the stream dimensions to the useCameraCanvas hook', () => {
     const { unmount } = render(<Camera />);
 

packages/public/camera-web/test/Camera/hooks/useCameraPreview.test.tsx

@@ -31,7 +31,6 @@ describe('useCameraPreview hook', () => {
       expect(getMediaConstraints).toHaveBeenCalledWith(undefined);
     });
     const options: CameraConfig = {
-      deviceId: 'test-id',
       facingMode: CameraFacingMode.USER,
       resolution: CameraResolution.QHD_2K,
     };

packages/public/camera-web/test/Camera/hooks/useUserMedia.test.ts

@@ -1,12 +1,17 @@
 import { useMonitoring } from '@monkvision/monitoring';
 
 jest.mock('@monkvision/monitoring');
+jest.mock('@monkvision/common', () => ({
+  ...jest.requireActual('@monkvision/common'),
+  isMobileDevice: jest.fn(() => false),
+}));
 
 import { act, waitFor } from '@testing-library/react';
 import { renderHook } from '@testing-library/react-hooks';
 import { UserMediaErrorType } from '../../../src';
 import { InvalidStreamErrorName, useUserMedia } from '../../../src/Camera/hooks';
 import { GetUserMediaMock, mockGetUserMedia } from '../../mocks';
+import { isMobileDevice } from '@monkvision/common';
 
 describe('useUserMedia hook', () => {
   let gumMock: GetUserMediaMock | null = null;
@@ -125,11 +130,13 @@ describe('useUserMedia hook', () => {
         kind: 'video',
         applyConstraints: jest.fn(() => Promise.resolve(undefined)),
         getSettings: jest.fn(() => ({ width: 456, height: 123 })),
+        stop: jest.fn(),
       },
       {
         kind: 'video',
         applyConstraints: jest.fn(() => Promise.resolve(undefined)),
         getSettings: jest.fn(() => ({ width: 456, height: 123 })),
+        stop: jest.fn(),
       },
     ] as unknown as MediaStreamTrack[];
     mockGetUserMedia({ tracks });
@@ -159,6 +166,7 @@ describe('useUserMedia hook', () => {
           kind: 'video',
           applyConstraints: jest.fn(() => Promise.resolve(undefined)),
           getSettings: jest.fn(() => invalidSettings[i]),
+          stop: jest.fn(),
         },
       ] as unknown as MediaStreamTrack[];
       mockGetUserMedia({ tracks });
@@ -233,7 +241,7 @@ describe('useUserMedia hook', () => {
     await waitFor(() => {
       expect(result.current.error).toBeNull();
       expect(result.current.stream).toEqual(mock.stream);
-      expect(mock.getUserMediaSpy).toHaveBeenCalledTimes(2);
+      expect(mock.getUserMediaSpy).toHaveBeenCalledTimes(3);
     });
     unmount();
   });
@@ -271,4 +279,54 @@ describe('useUserMedia hook', () => {
     });
     unmount();
   });
+
+  it('should switch the dimensions if the device is mobile', async () => {
+    const userAgentGetter = jest.spyOn(window.navigator, 'userAgent', 'get');
+    userAgentGetter.mockReturnValue('iphone');
+    const isMobileDeviceMock = isMobileDevice as jest.Mock;
+    isMobileDeviceMock.mockReturnValue(true);
+    const constraints: MediaStreamConstraints = {
+      audio: false,
+      video: { width: 123, height: 456 },
+    };
+    const { result, unmount } = renderHook(useUserMedia, {
+      initialProps: constraints,
+    });
+    await waitFor(() => {
+      expect(result.current.dimensions).toEqual({
+        height: 456,
+        width: 123,
+      });
+    });
+    unmount();
+  });
+
+  it('should filter the video constraints by removing: Telephoto and wide camera', async () => {
+    const userAgentGetter = jest.spyOn(window.navigator, 'userAgent', 'get');
+    userAgentGetter.mockReturnValue('iphone');
+    const constraints: MediaStreamConstraints = {
+      audio: false,
+      video: { width: 123, height: 456 },
+    };
+    gumMock?.enumerateDevicesSpy.mockResolvedValue([
+      { kind: 'videoinput', label: 'Front Camera', deviceId: 'frontDeviceId' },
+      { kind: 'videoinput', label: 'Rear Camera', deviceId: 'rearDeviceId' },
+      { kind: 'videoinput', label: 'Wide Angle Camera', deviceId: 'wideDeviceId' },
+      { kind: 'videoinput', label: 'Telephoto Angle Camera', deviceId: 'wideDeviceId' },
+    ]);
+    const { unmount } = renderHook(useUserMedia, {
+      initialProps: constraints,
+    });
+    await waitFor(() => {
+      expect(gumMock?.getUserMediaSpy).toHaveBeenCalledWith({
+        audio: false,
+        video: {
+          width: 123,
+          height: 456,
+          deviceId: { exact: ['frontDeviceId', 'rearDeviceId'] },
+        },
+      });
+    });
+    unmount();
+  });
 });

packages/public/camera-web/test/Camera/hooks/utils/getMediaConstraints.test.ts

@@ -20,21 +20,6 @@ const EXPECTED_CAMERA_RESOLUTION_SIZES: {
 };
 
 describe('useMediaConstraints hook', () => {
-  it('should properly map the deviceId option', () => {
-    const deviceId = 'test-id';
-    const constraints = getMediaConstraints({
-      deviceId,
-      facingMode: CameraFacingMode.ENVIRONMENT,
-      resolution: CameraResolution.UHD_4K,
-    });
-
-    expect(constraints).toEqual(
-      expect.objectContaining({
-        video: expect.objectContaining({ deviceId }),
-      }),
-    );
-  });
-
   Object.values(CameraFacingMode).forEach((facingMode) =>
     it(`should properly map the '${facingMode}' facingMode option`, () => {
       const constraints = getMediaConstraints({ facingMode, resolution: CameraResolution.UHD_4K });

packages/public/camera-web/test/Camera/hooks/utils/getValidCameraDeviceIds.test.ts

@@ -0,0 +1,24 @@
+import { getValidCameraDeviceIds } from '../../../../src/Camera/hooks/utils';
+import { mockGetUserMedia } from '../../../mocks';
+
+describe('getValidCameraDeviceIds util function', () => {
+  it('should return valid camera device IDs based on constraints', async () => {
+    const gumMock = mockGetUserMedia();
+    const devices = [
+      { kind: 'videoinput', label: 'Front Camera', deviceId: 'frontDeviceId' },
+      { kind: 'videoinput', label: 'Rear Camera', deviceId: 'rearDeviceId' },
+      { kind: 'videoinput', label: 'Wide Angle Camera', deviceId: 'wideDeviceId' },
+      { kind: 'videoinput', label: 'Telephoto Angle Camera', deviceId: 'telephotoDeviceId' },
+    ];
+    gumMock?.enumerateDevicesSpy.mockResolvedValue(devices);
+    const constraints: MediaStreamConstraints = {
+      audio: false,
+      video: { width: 123, height: 456 },
+    };
+    const validDeviceIds = await getValidCameraDeviceIds(constraints);
+
+    expect(validDeviceIds).toEqual([devices[0].deviceId, devices[1].deviceId]); // Adjust this expectation based on your logic
+    expect(gumMock.getUserMediaSpy).toHaveBeenCalledWith(constraints);
+    expect(gumMock.enumerateDevicesSpy).toHaveBeenCalled();
+  });
+});