feat: ocr text for translating

Author: geekdada

src/common/types.ts

@@ -4,6 +4,8 @@ export interface Config {
   token: string
   targetLang: SupportLanguageKeys
   region: APIRegions
+  ocrSecretId?: string
+  ocrSecretKey?: string
 }
 
 export type SupportLanguageKeys = keyof typeof supportedLanguages

src/components/svg/CursorClick.tsx

@@ -0,0 +1,20 @@
+import React from 'react'
+
+export default function CursorClick(): React.ReactElement {
+  return (
+    <svg
+      xmlns="http://www.w3.org/2000/svg"
+      fill="none"
+      width="16"
+      height="16"
+      viewBox="0 0 24 24"
+      stroke="currentColor">
+      <path
+        strokeLinecap="round"
+        strokeLinejoin="round"
+        strokeWidth={2}
+        d="M15 15l-2 5L9 9l11 4-5 2zm0 0l5 5M7.188 2.239l.777 2.897M5.136 7.965l-2.898-.777M13.95 4.05l-2.122 2.122m-5.657 5.656l-2.12 2.122"
+      />
+    </svg>
+  )
+}

src/components/svg/LoadingCircle.tsx

@@ -0,0 +1,30 @@
+import React, { SVGProps } from 'react'
+import tw, { css } from 'twin.macro'
+
+const LoadingCircle: React.FC<SVGProps<SVGSVGElement>> = (props) => {
+  return (
+    <svg
+      {...props}
+      xmlns="http://www.w3.org/2000/svg"
+      fill="none"
+      width="16"
+      height="16"
+      viewBox="0 0 24 24">
+      <circle
+        tw="opacity-25"
+        cx="12"
+        cy="12"
+        r="10"
+        stroke="currentColor"
+        strokeWidth="4"
+      />
+      <path
+        tw="opacity-75"
+        fill="currentColor"
+        d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"
+      />
+    </svg>
+  )
+}
+
+export default LoadingCircle

src/global.d.ts

@@ -32,3 +32,25 @@ declare module 'rangy' {
 
   export = rangy
 }
+
+declare module 'react-resizable' {
+  import { Axis, ResizeCallbackData, ResizeHandle } from 'react-resizable'
+
+  export interface ResizableProps {
+    className?: string
+    width: number
+    height: number
+    handle?: React.ReactNode | ((resizeHandle: ResizeHandle) => React.ReactNode)
+    handleSize?: [number, number]
+    lockAspectRatio?: boolean
+    axis?: Axis
+    minConstraints?: [number, number]
+    maxConstraints?: [number, number]
+    onResizeStop?: (e: React.SyntheticEvent, data: ResizeCallbackData) => any
+    onResizeStart?: (e: React.SyntheticEvent, data: ResizeCallbackData) => any
+    onResize?: (e: React.SyntheticEvent, data: ResizeCallbackData) => any
+    draggableOpts?: any
+    resizeHandles?: ResizeHandle[]
+    style?: Record<string, any>
+  }
+}

src/pages/Background/common/ocr-client.ts

@@ -0,0 +1,136 @@
+import defaultsDeep from 'lodash-es/defaultsDeep'
+import { SHA256, HmacSHA256, enc } from 'crypto-js'
+import axios from 'axios'
+
+export class TcRequestError extends Error {
+  code?: string
+
+  constructor(message: string, code?: string) {
+    super(message + (code ? ` [${code}]` : ''))
+    this.name = 'TcRequestError'
+    if (code) this.code = code
+  }
+}
+
+export class OcrClient {
+  private config: {
+    secretId: string
+    secretKey: string
+    region: string
+  }
+  private requestConfig = {
+    host: 'ocr.tencentcloudapi.com',
+    service: 'ocr',
+    version: '2018-11-19',
+    algorithm: 'TC3-HMAC-SHA256',
+    httpRequestMethod: 'POST',
+    canonicalUri: '/',
+    canonicalQueryString: '',
+    canonicalHeaders:
+      'content-type:application/json; charset=utf-8\nhost:ocr.tencentcloudapi.com\n',
+    signedHeaders: 'content-type;host',
+  }
+
+  constructor(config: { secretId: string; secretKey: string; host?: string }) {
+    this.config = defaultsDeep({}, config, {
+      region: 'ap-shanghai',
+    })
+  }
+
+  async request(data: { dataUrl: string }): Promise<string[]> {
+    const payload = {
+      ImageBase64: data.dataUrl,
+    }
+    const signature = this.signPayload(payload)
+    const headers = {
+      Authorization: signature.authorization,
+      'Content-Type': 'application/json; charset=UTF-8',
+      'X-TC-Action': 'GeneralBasicOCR',
+      'X-TC-Timestamp': signature.timestamp,
+      'X-TC-Version': this.requestConfig.version,
+      'X-TC-Region': this.config.region,
+      'X-TC-RequestClient': `WXAPP_SDK_OcrSDK_1.1.0`,
+    }
+
+    return axios
+      .request({
+        url: 'https://ocr.tencentcloudapi.com',
+        data: payload,
+        method: 'POST',
+        headers,
+        responseType: 'json',
+      })
+      .then((res) => {
+        if (res.data?.Response?.Error) {
+          const error = res.data.Response.Error
+          throw new TcRequestError(error.Message, error.Code)
+        }
+
+        if (!res.data?.Response?.TextDetections) {
+          throw new TcRequestError('没有数据返回')
+        }
+
+        const detections = res.data.Response.TextDetections
+        const result: string[] = []
+
+        detections.forEach(
+          (item: { DetectedText: string; AdvancedInfo: string }) => {
+            const advanceInfo: {
+              Parag: {
+                ParagNo: number
+              }
+            } = JSON.parse(item.AdvancedInfo)
+            const index = advanceInfo.Parag.ParagNo - 1
+
+            if (result[index]) {
+              result[index] += ` ${item.DetectedText}`
+            } else {
+              result.push(item.DetectedText)
+            }
+          },
+        )
+
+        return result
+      })
+  }
+
+  private signPayload(
+    payload: Record<string, any>,
+  ): { authorization: string; timestamp: number } {
+    const hashedRequestPayload = SHA256(JSON.stringify(payload))
+    const canonicalRequest = [
+      this.requestConfig.httpRequestMethod,
+      this.requestConfig.canonicalUri,
+      this.requestConfig.canonicalQueryString,
+      this.requestConfig.canonicalHeaders,
+      this.requestConfig.signedHeaders,
+      hashedRequestPayload,
+    ].join('\n')
+    const t = new Date()
+    const date = t.toISOString().substr(0, 10)
+    const timestamp = Math.round(t.getTime() / 1000)
+    const credentialScope = `${date}/${this.requestConfig.service}/tc3_request`
+    const hashedCanonicalRequest = SHA256(canonicalRequest)
+    const stringToSign = [
+      this.requestConfig.algorithm,
+      timestamp,
+      credentialScope,
+      hashedCanonicalRequest,
+    ].join('\n')
+
+    const secretDate = HmacSHA256(date, `TC3${this.config.secretKey}`)
+    const secretService = HmacSHA256(this.requestConfig.service, secretDate)
+    const secretSigning = HmacSHA256('tc3_request', secretService)
+
+    const signature = enc.Hex.stringify(HmacSHA256(stringToSign, secretSigning))
+
+    return {
+      authorization:
+        `${this.requestConfig.algorithm} ` +
+        `Credential=${this.config.secretId}/${credentialScope}, ` +
+        `SignedHeaders=${this.requestConfig.signedHeaders}, ` +
+        `Signature=${signature}`,
+      timestamp,
+    }
+  }
+}

src/pages/Background/common/server.ts

@@ -4,7 +4,9 @@ import { createServer } from 'connect.io'
 import Client from '../../../common/api'
 import logger from '../../../common/logger'
 import { Config } from '../../../common/types'
+import { OcrClient } from './ocr-client'
 import { Handler } from './types'
+import { cropImage } from './utils'
 
 const server = createServer()
 
@@ -46,8 +48,83 @@ const onTranslate: Handler<{
   })
 }
 
+const onScreenshot: Handler<{
+  x: number
+  y: number
+  width: number
+  height: number
+  clientWidth: number
+  clientHeight: number
+  clientPixelRatio: number
+}> = (payload, resolve, reject) => {
+  ;(async () => {
+    logger.debug(
+      {
+        payload,
+      },
+      'receive screenshot payload',
+    )
+
+    const dataUrl: string = await cc(chrome.tabs, 'captureVisibleTab', null, {
+      quality: 75,
+    })
+
+    resolve({
+      dataUrl: await cropImage(dataUrl, {
+        ...payload,
+        imageWidth: payload.clientWidth,
+        imageHeight: payload.clientHeight,
+        imageRatio: payload.clientPixelRatio,
+      }),
+    })
+  })().catch((err) => {
+    logger.error({
+      err,
+    })
+    reject({
+      message: err.message,
+    })
+  })
+}
+
+const onOCR: Handler<{
+  dataUrl: string
+}> = (payload, resolve, reject) => {
+  ;(async () => {
+    logger.debug(
+      {
+        payload,
+      },
+      'receive ocr payload',
+    )
+
+    const config: Config = await cc(chrome.storage.sync, 'get')
+
+    if (!config.ocrSecretId || !config.ocrSecretKey) {
+      return
+    }
+
+    const client = new OcrClient({
+      secretId: config.ocrSecretId,
+      secretKey: config.ocrSecretKey,
+    })
+    const data = await client.request({ dataUrl: payload.dataUrl })
+
+    resolve(data)
+  })().catch((err) => {
+    logger.error({
+      err,
+    })
+    reject({
+      message: err.message,
+    })
+  })
+}
+
 server.on('connect', (client) => {
   client.on('translate', onTranslate)
+  client.on('screenshot', onScreenshot)
+  client.on('ocr', onOCR)
 })
 
 export default server

src/pages/Background/common/utils.ts

@@ -1,5 +1,7 @@
+import { data } from 'autoprefixer'
 import cc from 'chrome-call'
 import { createClient } from 'connect.io'
+
 import logger from '../../../common/logger'
 
 export const openExtension = (): void => {
@@ -15,3 +17,54 @@ export const openExtension = (): void => {
       })
     })
 }
+
+export const cropImage = async (
+  dataUrl: string,
+  config: {
+    x: number
+    y: number
+    width: number
+    height: number
+    imageWidth: number
+    imageHeight: number
+    imageRatio: number
+  },
+): Promise<string> => {
+  const croppedCanvas = await new Promise<HTMLCanvasElement>(
+    (resolve, reject) => {
+      const canvas = document.createElement('canvas')
+      const img = new Image()
+
+      canvas.width = config.width
+      canvas.height = config.height
+      img.onload = () => {
+        const ctx = canvas.getContext('2d')
+
+        canvas.width = config.width
+        canvas.height = config.height
+
+        ctx?.drawImage(
+          img,
+          config.x * config.imageRatio,
+          config.y * config.imageRatio,
+          config.width * config.imageRatio,
+          config.height * config.imageRatio,
+          0,
+          0,
+          config.width,
+          config.height,
+        )
+
+        resolve(canvas)
+      }
+
+      img.onerror = () => {
+        reject(new Error('Failed to load image'))
+      }
+
+      img.src = dataUrl
+    },
+  )
+
+  return croppedCanvas.toDataURL('image/jpeg')
+}

src/pages/Content/common/types.ts

@@ -11,7 +11,7 @@ export interface TextSelection {
 
 export interface TranslateJob {
   id: string
-  anchorId: string
   text: string
-  sourceLang?: SupportLanguageKeys
+  anchorId?: string
+  sourceLang?: string
 }