Always display camera preview with highest res possible (#685)

* Always display camera preview with highest res possible

* Small tweaks

Author: souyahia-monk

apps/monk-test-app/src/views/TestView/TestView.css

@@ -7,3 +7,10 @@
   justify-content: center;
   color: white;
 }
+
+.select-container {
+  position: fixed;
+  top: 50px;
+  left: 50px;
+  z-index: 9999;
+}

apps/monk-test-app/src/views/TestView/TestView.tsx

@@ -1,52 +1,39 @@
-import React, { useState } from 'react';
-import { PhotoCapture } from '@monkvision/inspection-capture-web';
-import { sights } from '@monkvision/sights';
+import { Camera, CameraResolution, MonkPicture, SimpleCameraHUD } from '@monkvision/camera-web';
 import './TestView.css';
-
-const captureSights = [
-  sights['haccord-8YjMcu0D'],
-  sights['haccord-DUPnw5jj'],
-  sights['haccord-hsCc_Nct'],
-  sights['haccord-GQcZz48C'],
-  sights['haccord-QKfhXU7o'],
-  sights['haccord-mdZ7optI'],
-  sights['haccord-bSAv3Hrj'],
-  sights['haccord-W-Bn3bU1'],
-  sights['haccord-GdWvsqrm'],
-  sights['haccord-ps7cWy6K'],
-  sights['haccord-Jq65fyD4'],
-  sights['haccord-OXYy5gET'],
-  sights['haccord-5LlCuIfL'],
-  sights['haccord-Gtt0JNQl'],
-  sights['haccord-cXSAj2ez'],
-  sights['haccord-KN23XXkX'],
-  sights['haccord-Z84erkMb'],
-];
-
-const inspectionId = 'b072ff42-6244-ef3b-b018-5d3d6562c1bd';
-
-const apiConfig = {
-  apiDomain: 'api.preview.monk.ai/v1',
-  authToken:
-    'eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IjNLUnpaNm01WDFzOWFBZWRudnBrWSJ9.eyJpc3MiOiJodHRwczovL2lkcC5wcmV2aWV3Lm1vbmsuYWkvIiwic3ViIjoiZ29vZ2xlLW9hdXRoMnwxMDY5MzYxMTEwMDU4MDYxODA1NTYiLCJhdWQiOlsiaHR0cHM6Ly9hcGkubW9uay5haS92MS8iLCJodHRwczovL21vbmstcHJldmlldy5ldS5hdXRoMC5jb20vdXNlcmluZm8iXSwiaWF0IjoxNzA3NDcxNzU5LCJleHAiOjE3MDc0Nzg5NTksImF6cCI6InNvWjdQMmM2YjlJNWphclFvUnJoaDg3eDlUcE9TYUduIiwic2NvcGUiOiJvcGVuaWQgcHJvZmlsZSBlbWFpbCIsInBlcm1pc3Npb25zIjpbIm1vbmtfY29yZV9hcGk6Y29tcGxpYW5jZXMiLCJtb25rX2NvcmVfYXBpOmRhbWFnZV9kZXRlY3Rpb24iLCJtb25rX2NvcmVfYXBpOmRhc2hib2FyZF9vY3IiLCJtb25rX2NvcmVfYXBpOmltYWdlc19vY3IiLCJtb25rX2NvcmVfYXBpOmluc3BlY3Rpb25zOmNyZWF0ZSIsIm1vbmtfY29yZV9hcGk6aW5zcGVjdGlvbnM6ZGVsZXRlIiwibW9ua19jb3JlX2FwaTppbnNwZWN0aW9uczpkZWxldGVfYWxsIiwibW9ua19jb3JlX2FwaTppbnNwZWN0aW9uczpkZWxldGVfb3JnYW5pemF0aW9uIiwibW9ua19jb3JlX2FwaTppbnNwZWN0aW9uczpyZWFkIiwibW9ua19jb3JlX2FwaTppbnNwZWN0aW9uczpyZWFkX2FsbCIsIm1vbmtfY29yZV9hcGk6aW5zcGVjdGlvbnM6cmVhZF9vcmdhbml6YXRpb24iLCJtb25rX2NvcmVfYXBpOmluc3BlY3Rpb25zOnVwZGF0ZSIsIm1vbmtfY29yZV9hcGk6aW5zcGVjdGlvbnM6dXBkYXRlX2FsbCIsIm1vbmtfY29yZV9hcGk6aW5zcGVjdGlvbnM6dXBkYXRlX29yZ2FuaXphdGlvbiIsIm1vbmtfY29yZV9hcGk6aW5zcGVjdGlvbnM6d3JpdGUiLCJtb25rX2NvcmVfYXBpOmluc3BlY3Rpb25zOndyaXRlX2FsbCIsIm1vbmtfY29yZV9hcGk6aW5zcGVjdGlvbnM6d3JpdGVfb3JnYW5pemF0aW9uIiwibW9ua19jb3JlX2FwaTpyZXBhaXJfZXN0aW1hdGUiLCJtb25rX2NvcmVfYXBpOnVzZXJzOnJlYWQiLCJtb25rX2NvcmVfYXBpOnVzZXJzOnJlYWRfYWxsIiwibW9ua19jb3JlX2FwaTp1c2VyczpyZWFkX29yZ2FuaXphdGlvbiIsIm1vbmtfY29yZV9hcGk6dXNlcnM6dXBkYXRlIiwibW9ua19jb3JlX2FwaTp1c2Vyczp1cGRhdGVfYWxsIiwibW9ua19jb3JlX2FwaTp1c2Vyczp1cGRhdGVfb3JnYW5pemF0aW9uIiwibW9ua19jb3JlX2FwaTp1c2Vyczp3cml0ZSIsIm1vbmtfY29yZV9hcGk6dXNlcnM6d3JpdGVfYWxsIiwibW9ua19jb3JlX2FwaTp1c2Vyczp3cml0ZV9vcmdhbml6YXRpb24iLCJtb25rX2NvcmVfYXBpOndoZWVsX2FuYWx5c2lzIl19.m38G5NaCD_pcGptJdLPa_TVtD08yRY9qdp-5pCELd8Ekzma0kCWMctwHvlrv2OsjynlXyAutIK2uhDMrdLdnPk_6bU4rZheej2s0obXaXqZCUTbUOh8scL-81tbH_ZlKN3oSXfqUVMnwvpa1bnXZHmjeHi2e3bhvjxW-Jg5DrBB9gNfstxK0hugrlxtNL96y6ImEITOxOMEbURYGwOLQQtLkRqFo7AeZCu-_w6UbtFZfLJc5FlsWpTKy7I3_xMynzDGGaADRQeyazL_7DfemCb_VPXkR9aV67tF4pt6jevCetYI5CfN5X2JJrp7XNES_M2d7wGdJl5C6lbBPs3n3SA',
-};
+import { useState } from 'react';
 
 export function TestView() {
-  const [complete, setComplete] = useState(false);
+  const [resolution, setResolution] = useState(CameraResolution.UHD_4K);
+
+  const handlePictureTaken = (picture: MonkPicture) => {
+    const link = document.createElement('a');
+    link.href = picture.uri;
+    const now = new Date();
+    link.download = `pic-${resolution}-${now.getHours()}-${now.getMinutes()}-${now.getSeconds()}.jpg`;
+    document.body.appendChild(link);
+    link.click();
+    document.body.removeChild(link);
+  };
 
   return (
     <div className='test-view-container'>
-      {complete ? (
-        'Inspection Complete!'
-      ) : (
-        <PhotoCapture
-          sights={captureSights}
-          inspectionId={inspectionId}
-          apiConfig={apiConfig}
-          onComplete={() => setComplete(true)}
-          onClose={() => console.log('coucou')}
-        />
-      )}
+      <Camera
+        HUDComponent={SimpleCameraHUD}
+        resolution={resolution}
+        onPictureTaken={handlePictureTaken}
+      />
+      <div className='select-container'>
+        <select
+          value={resolution}
+          onChange={(e) => setResolution(e.target.value as CameraResolution)}
+        >
+          {Object.values(CameraResolution).map((res) => (
+            <option key={res} value={res}>
+              {res}
+            </option>
+          ))}
+        </select>
+      </div>
     </div>
   );
 }

packages/public/camera-web/README.md

@@ -33,13 +33,9 @@ function MyCameraPreview() {
 }
 ```
 
-## Camera constraints
-The resolution quality of the camera of the Camera video stream that is fetched from the user's device is configurable
-by passing it as a prop to the Camera component. Note that device selection (selecting which Camera will be used when
-the device has many available) is disabled for now. This is because this is instead handled automatically by the
-component in order to prevent unusable cameras (zoomed ones for instance) to be used.
-
-Example of how to configure the resolution of the Camera :
+## Camera resolution
+The resolution quality of the pictures taken by the Camera component is configurable by passing it as a prop to the
+Camera component :
 
 ```tsx
 import { Camera, CameraResolution } from '@monkvision/camera-web';
@@ -49,20 +45,15 @@ function MyCameraPreview() {
 }
 ```
 
-For more details on the camera constraints options, see the *API* section below.
-
 Notes :
-- When the camera constraints are updated, the video stream is automatically updated without asking the user for
-  permissions another time
-- Only the resolutions available in the `CameraResolution` enum are allowed for better results with our AI models
-- The resolutions available in the `CameraResolution` enum are all in the 16:9 format
-- Device selection (selecting which Camera will be used when the device has many available) is disabled for now. This is
-  because this is instead handled automatically by the component in order to prevent unusable cameras (zoomed ones for
-  instance) to be used.
-- If no device meets the given requirements, the device with the closest match will be used :
-  - If the needed resolution is too high, the highest resolution will be used : this means that asking for the
-    `CameraResolution.UHD_4K` resolution is a good way to get the highest resolution possible
-  - If the needed resolution is too low the browser will crop and scale the camera's feed to meet the requirements
+- This option does not affect the resolution of the Camera preview : the preview will always use the highest
+  resolution available on the current device.
+- If the specified resolution is not equal to the one used by the device's native camera, the pictures taken will be
+  scaled to fit the requirements.
+- The resolutions available in the `CameraResolution` enum are all in the 16:9 format.
+- If the aspect ratio of the specified resolution differs from the one of the device's camera, pictures taken will
+  always have the same aspect ratio as the native camera one, and will be scaled in a way to make sure that neither the
+  width, nor the height of the output picture will exceed the dimensions of the specified resolution.
 
 ## Compression options
 When pictures are taken by the camera, they are compressed and encoded. The compression format and quality can be
@@ -169,7 +160,7 @@ Main component exported by this package, displays a Camera preview and the given
 ### Props
 | Prop           | Type                           | Description                                                                                                                     | Required | Default Value                  |
 |----------------|--------------------------------|---------------------------------------------------------------------------------------------------------------------------------|----------|--------------------------------|
-| resolution     | CameraResolution               | Resolution of the camera to use.                                                                                                |          | `CameraResolution.UHD_4K`      |
+| resolution     | CameraResolution               | Resolution of the pictures taken by the camera. This does not affect the resolution of the Camera preview.                      |          | `CameraResolution.UHD_4K`      |
 | format         | CompressionFormat              | The compression format used to compress images taken by the camera.                                                             |          | `CompressionFormat.JPEG`       |
 | quality        | number                         | The image quality when using a compression format that supports lossy compression. From 0 (lowest quality) to 1 (best quality). |          | `0.8`                          |
 | HUDComponent   | CameraHUDComponent<T>          | The camera HUD component to display on top of the camera preview.                                                               |          |                                |

packages/public/camera-web/src/Camera/Camera.tsx

@@ -1,7 +1,6 @@
 import React, { useMemo } from 'react';
 import { AllOrNone, RequiredKeys } from '@monkvision/types';
 import {
-  CameraConfig,
   CameraFacingMode,
   CameraResolution,
   CompressionFormat,
@@ -47,10 +46,31 @@ export type HUDConfigProps<T extends object> = RequiredKeys<T> extends never
 /**
  * Props given to the Camera component. The generic T type corresponds to the prop types of the HUD.
  */
-export type CameraProps<T extends object> = Partial<Pick<CameraConfig, 'resolution'>> &
-  Partial<CompressionOptions> &
+export type CameraProps<T extends object> = Partial<CompressionOptions> &
   CameraEventHandlers &
   HUDConfigProps<T> & {
+    /**
+     * This option specifies the resolution of the pictures taken by the Camera. This option does not affect the
+     * resolution of the Camera preview (it will always be the highest resolution possible). If the specified resolution
+     * is not equal to the one used by the device's native camera, the pictures taken will be scaled to fit the
+     * requirements. Note that if the aspect ratio of the specified resolution differs from the one of the device's
+     * camera, pictures taken will always have the same aspect ratio as the native camera one, and will be scaled in a way
+     * to make sure that neither the width, nor the height of the output picture will exceed the dimensions of the
+     * specified resolution.
+     *
+     * Note: If the specified resolution is higher than the best resolution available on the current device, output
+     * pictures will only be scaled up to the specified resolution if the `allowImageUpscaling` property is set to `true`.
+     *
+     * @default `CameraResolution.UHD_4K`
+     */
+    resolution?: CameraResolution;
+    /**
+     * When the native resolution of the device Camera is smaller than the resolution asked in the `resolution` prop,
+     * resulting pictures will only be scaled up if this property is set to `true`.
+     *
+     * @default `false`
+     */
+    allowImageUpscaling?: boolean;
     /**
      * Additional monitoring config that can be provided to the Camera component.
      */
@@ -71,20 +91,32 @@ export function Camera<T extends object>({
   resolution = CameraResolution.UHD_4K,
   format = CompressionFormat.JPEG,
   quality = 0.8,
+  allowImageUpscaling = false,
   HUDComponent,
   hudProps,
   monitoring,
   onPictureTaken,
 }: CameraProps<T>) {
   const {
     ref: videoRef,
-    dimensions,
+    dimensions: streamDimensions,
     error,
     retry,
     isLoading: isPreviewLoading,
-  } = useCameraPreview({ resolution, facingMode: CameraFacingMode.ENVIRONMENT });
-  const { ref: canvasRef } = useCameraCanvas({ dimensions });
-  const { takeScreenshot } = useCameraScreenshot({ videoRef, canvasRef, dimensions });
+  } = useCameraPreview({
+    resolution: CameraResolution.UHD_4K,
+    facingMode: CameraFacingMode.ENVIRONMENT,
+  });
+  const { ref: canvasRef, dimensions: canvasDimensions } = useCameraCanvas({
+    resolution,
+    streamDimensions,
+    allowImageUpscaling,
+  });
+  const { takeScreenshot } = useCameraScreenshot({
+    videoRef,
+    canvasRef,
+    dimensions: canvasDimensions,
+  });
   const { compress } = useCompression({ canvasRef, options: { format, quality } });
   const { takePicture, isLoading: isTakePictureLoading } = useTakePicture({
     compress,
@@ -112,7 +144,7 @@ export function Camera<T extends object>({
 
   return HUDComponent ? (
     <HUDComponent
-      handle={{ takePicture, error, retry, isLoading, dimensions }}
+      handle={{ takePicture, error, retry, isLoading, dimensions: streamDimensions }}
       cameraPreview={cameraPreview}
       {...((hudProps ?? {}) as T)}
     />

packages/public/camera-web/src/Camera/hooks/useCameraCanvas.ts

@@ -1,14 +1,24 @@
-import { RefObject, useEffect, useRef } from 'react';
+import { RefObject, useEffect, useMemo, useRef } from 'react';
 import { PixelDimensions } from '@monkvision/types';
+import { CameraResolution, getResolutionDimensions } from './utils';
 
 /**
  * Object used to configure the camera canvas.
  */
 export interface CameraCanvasConfig {
+  /**
+   * The resolution of the pictures taken asked by the user of the Camera component.
+   */
+  resolution: CameraResolution;
+  /**
+   * Boolean indicating if the Camera component should allow image upscaling when the asked resolution is bigger than
+   * the one of the device Camera.
+   */
+  allowImageUpscaling: boolean;
   /**
    * The dimensions of the video stream.
    */
-  dimensions: PixelDimensions | null;
+  streamDimensions: PixelDimensions | null;
 }
 
 /**
@@ -19,13 +29,65 @@ export interface CameraCanvasHandle {
    * The ref to the canvas element. Forward this ref to the <canvas> tag to set it up.
    */
   ref: RefObject<HTMLCanvasElement>;
+  /**
+   * The dimensions of the canvas.
+   */
+  dimensions: PixelDimensions | null;
+}
+
+/**
+ * This function is used to calculate the dimensions of the canvas that will be used to draw the image, thus also
+ * calculating the output dimensions of the image itself, respecting the following logic :
+ * - If the aspect ratio of the stream and constraints are the same, we simply scale the stream to make it fit the
+ * constraints. Note that if `allowImageUpscaling` is `false`, and the stream is smaller than the constraints, we don't
+ * scale "up" the stream image, and we simply return the stream dimensions.
+ * - If the aspect ratio of the stream is different from the one specified in the constraints, the logic is the same,
+ * but the output aspect ratio will be the same one as the stream. The stream dimensions will simply be scaled
+ * following the same logic as the previous point, making sure that neither the width nor the height of the canvas will
+ * exceed the ones described by the constraints.
+ */
+function getCanvasDimensions({
+  resolution,
+  streamDimensions,
+  allowImageUpscaling,
+}: CameraCanvasConfig): PixelDimensions | null {
+  if (!streamDimensions) {
+    return null;
+  }
+  const isPortrait = streamDimensions.width < streamDimensions.height;
+  const constraintsDimensions = getResolutionDimensions(resolution, isPortrait);
+  const streamRatio = streamDimensions.width / streamDimensions.height;
+
+  if (
+    constraintsDimensions.width > streamDimensions.width &&
+    constraintsDimensions.height > streamDimensions.height &&
+    !allowImageUpscaling
+  ) {
+    return {
+      width: streamDimensions.width,
+      height: streamDimensions.height,
+    };
+  }
+  const fitToHeight = constraintsDimensions.width / streamRatio > constraintsDimensions.height;
+  return {
+    width: fitToHeight ? constraintsDimensions.height * streamRatio : constraintsDimensions.width,
+    height: fitToHeight ? constraintsDimensions.height : constraintsDimensions.width / streamRatio,
+  };
 }
 
 /**
  * Custom hook used to manage the camera <canvas> element used to take video screenshots and encode images.
  */
-export function useCameraCanvas({ dimensions }: CameraCanvasConfig): CameraCanvasHandle {
+export function useCameraCanvas({
+  resolution,
+  streamDimensions,
+  allowImageUpscaling,
+}: CameraCanvasConfig): CameraCanvasHandle {
   const ref = useRef<HTMLCanvasElement>(null);
+  const dimensions = useMemo(
+    () => getCanvasDimensions({ resolution, streamDimensions, allowImageUpscaling }),
+    [resolution, streamDimensions],
+  );
 
   useEffect(() => {
     if (dimensions && ref.current) {
@@ -34,5 +96,5 @@ export function useCameraCanvas({ dimensions }: CameraCanvasConfig): CameraCanva
     }
   }, [dimensions]);
 
-  return { ref };
+  return { ref, dimensions };
 }

packages/public/camera-web/src/Camera/hooks/utils/getMediaContraints.ts

@@ -1,3 +1,5 @@
+import { PixelDimensions } from '@monkvision/types';
+
 /**
  * Enumeration of the facing modes for the camera constraints.
  */
@@ -50,8 +52,8 @@ export enum CameraResolution {
   UHD_4K = '4K',
 }
 
-const CAMERA_RESOLUTION_SIZES: {
-  [key in CameraResolution]: { width: number; height: number };
+const CAMERA_RESOLUTION_DIMENSIONS: {
+  [key in CameraResolution]: PixelDimensions;
 } = {
   [CameraResolution.QNHD_180P]: { width: 320, height: 180 },
   [CameraResolution.NHD_360P]: { width: 640, height: 360 },
@@ -68,8 +70,6 @@ export interface CameraConfig {
   /**
    * Specifies which camera to use if the devices has a front and a rear camera. If the device does not have a camera
    * meeting the requirements, the closest one will be used.
-   *
-   * @default `CameraFacingMode.ENVIRONMENT`
    */
   facingMode: CameraFacingMode;
   /**
@@ -78,12 +78,24 @@ export interface CameraConfig {
    * - The Monk Camera package will always try to fetch a stream with a 16:9 resolution format.
    * - The implementation of the algorithm used to choose the closest camera can differ between browsers, and if the
    * exact requirements can't be met, the resulting stream's quality can differ between browsers.
-   *
-   * @default `CameraResolution.UHD_4K`
    */
   resolution: CameraResolution;
 }
 
+/**
+ * Utility function that returns the dimensions in pixels of the given `CameraResolution`.
+ */
+export function getResolutionDimensions(
+  resolution: CameraResolution,
+  isPortrait = false,
+): PixelDimensions {
+  const dimensions = CAMERA_RESOLUTION_DIMENSIONS[resolution];
+  return {
+    width: isPortrait ? dimensions.height : dimensions.width,
+    height: isPortrait ? dimensions.width : dimensions.height,
+  };
+}
+
 /**
  * This function is used by the Monk Camera package in order to add a layer of abstraction to the media constraints
  * passed to the `useUserMedia` hook. It takes an optional `CameraOptions` parameter and creates a
@@ -94,7 +106,7 @@ export interface CameraConfig {
  * @see useUserMedia
  */
 export function getMediaConstraints(config: CameraConfig): MediaStreamConstraints {
-  const { width, height } = CAMERA_RESOLUTION_SIZES[config.resolution];
+  const { width, height } = getResolutionDimensions(config.resolution);
 
   const video: MediaTrackConstraints = {
     width: { ideal: width },