feat: support get scf logs by cls

Author: yugasun

__tests__/cls.test.ts

@@ -1,4 +1,5 @@
 import { ClsDeployInputs, ClsDeployOutputs } from './../src/modules/cls/interface';
+import { Scf } from '../src';
 import { Cls } from '../src';
 import { sleep } from '@ygkit/request';
 
@@ -7,6 +8,7 @@ describe('Cls', () => {
     SecretId: process.env.TENCENT_SECRET_ID,
     SecretKey: process.env.TENCENT_SECRET_KEY,
   };
+  const scf = new Scf(credentials, process.env.REGION);
   const client = new Cls(credentials, process.env.REGION);
 
   let outputs: ClsDeployOutputs;
@@ -43,16 +45,37 @@ describe('Cls', () => {
     outputs = res;
   });
 
-  test('should remove cls success', async () => {
+  test('remove cls', async () => {
     await sleep(5000);
     await client.remove(outputs);
 
-    const detail = await client.cls.getLogset({
-      logset_id: outputs.logsetId,
+    const detail = await client.cls.getTopic({
+      topic_id: outputs.topicId,
     });
-    expect(detail.logset_id).toBeUndefined();
+
+    expect(detail.topicId).toBeUndefined();
     expect(detail.error).toEqual({
       message: expect.any(String),
     });
   });
+
+  test('search log', async () => {
+    await scf.invoke({
+      namespace: 'default',
+      functionName: 'serverless-unit-test',
+    });
+
+    await sleep(5000);
+
+    const res = await client.getLogList({
+      functionName: 'serverless-unit-test',
+      namespace: 'default',
+      qualifier: '$LATEST',
+      logsetId: '125d5cd7-caee-49ab-af9b-da29aa09d6ab',
+      topicId: 'e9e38c86-c7ba-475b-a852-6305880d2212',
+      interval: 3600,
+    });
+    console.log('logs', res);
+    expect(res).toBeInstanceOf(Array);
+  });
 });

__tests__/scf.test.ts

@@ -537,17 +537,6 @@ describe('Scf', () => {
       },
     ]);
   });
-  test('[remove cls] update', async () => {
-    await sleep(3000);
-    inputs.cls = {
-      logsetId: '',
-      topicId: '',
-    };
-    outputs = await scf.deploy(inputs);
-
-    expect(outputs.ClsLogsetId).toBe('');
-    expect(outputs.ClsTopicId).toBe('');
-  });
   test('invoke', async () => {
     const res = await scf.invoke({
       namespace: inputs.namespace,
@@ -567,6 +556,26 @@ describe('Scf', () => {
       RequestId: expect.any(String),
     });
   });
+  test('get function logs', async () => {
+    const logs = await scf.logs({
+      functionName: inputs.name,
+      namespace: inputs.namespace,
+    });
+
+    expect(logs).toBeInstanceOf(Array);
+  });
+  test('[remove cls] update', async () => {
+    await sleep(3000);
+    inputs.cls = {
+      logsetId: '',
+      topicId: '',
+    };
+    outputs = await scf.deploy(inputs);
+
+    expect(outputs.ClsLogsetId).toBe('');
+    expect(outputs.ClsTopicId).toBe('');
+  });
+
   test('remove', async () => {
     const res = await scf.remove({
       functionName: inputs.name,

package.json

@@ -83,11 +83,12 @@
   },
   "dependencies": {
     "@tencent-sdk/capi": "^1.1.8",
-    "@tencent-sdk/cls": "^0.1.7",
+    "@tencent-sdk/cls": "^0.1.13",
     "@types/jest": "^26.0.20",
     "@types/node": "^14.14.31",
     "@ygkit/request": "^0.1.8",
     "cos-nodejs-sdk-v5": "2.8.6",
+    "dayjs": "^1.10.4",
     "moment": "^2.29.1",
     "tencent-cloud-sdk": "^1.0.5",
     "type-fest": "^0.20.2"

src/modules/cls/index.ts

@@ -1,14 +1,20 @@
-import { CapiCredentials, RegionType } from './../interface';
 import { Cls as ClsClient } from '@tencent-sdk/cls';
+import dayjs, { Dayjs } from 'dayjs';
 import {
   ClsDelopyIndexInputs,
   ClsDeployInputs,
   ClsDeployLogsetInputs,
   ClsDeployOutputs,
   ClsDeployTopicInputs,
+  GetLogOptions,
+  GetLogDetailOptions,
+  LogContent,
 } from './interface';
+import { CapiCredentials, RegionType } from './../interface';
 import { ApiError } from '../../utils/error';
-import { createLogset, createTopic, updateIndex } from './utils';
+import { createLogset, createTopic, updateIndex, getSearchSql } from './utils';
+
+const TimeFormat = 'YYYY-MM-DD HH:mm:ss';
 
 export default class Cls {
   credentials: CapiCredentials;
@@ -192,4 +198,96 @@ export default class Cls {
 
     return {};
   }
+
+  async getLogList(data: GetLogOptions) {
+    const clsClient = new ClsClient({
+      region: this.region,
+      secretId: this.credentials.SecretId!,
+      secretKey: this.credentials.SecretKey!,
+      token: this.credentials.Token,
+      debug: false,
+    });
+
+    const { endTime, interval = 3600 } = data;
+    let startDate: Dayjs;
+    let endDate: Dayjs;
+
+    // 默认获取从当前到一个小时前时间段的日志
+    if (!endTime) {
+      endDate = dayjs();
+      startDate = endDate.add(-1, 'hour');
+    } else {
+      endDate = dayjs(endTime);
+      startDate = dayjs(endDate.valueOf() - Number(interval) * 1000);
+    }
+
+    const sql = getSearchSql({
+      ...data,
+      startTime: startDate.valueOf(),
+      endTime: endDate.valueOf(),
+    });
+    const searchParameters = {
+      logset_id: data.logsetId,
+      topic_ids: data.topicId,
+      start_time: startDate.format(TimeFormat),
+      end_time: endDate.format(TimeFormat),
+      // query_string 必须用 cam 特有的 url 编码方式
+      query_string: sql,
+      limit: data.limit || 10,
+      sort: 'desc',
+    };
+    const { results = [] } = await clsClient.searchLog(searchParameters);
+    const logs = [];
+    for (let i = 0, len = results.length; i < len; i++) {
+      const curReq = results[i];
+      const detailLog = await this.getLogDetail({
+        logsetId: data.logsetId,
+        topicId: data.topicId,
+        reqId: curReq.requestId,
+        startTime: startDate.format(TimeFormat),
+        endTime: endDate.format(TimeFormat),
+      });
+      curReq.message = (detailLog || [])
+        .map(({ content }: { content: string }) => {
+          try {
+            const info = JSON.parse(content) as LogContent;
+            if (info.SCF_Type === 'Custom') {
+              curReq.memoryUsage = info.SCF_MemUsage;
+              curReq.duration = info.SCF_Duration;
+            }
+            return info.SCF_Message;
+          } catch (e) {
+            return '';
+          }
+        })
+        .join('');
+      logs.push(curReq);
+    }
+    return logs;
+  }
+  async getLogDetail(data: GetLogDetailOptions) {
+    const clsClient = new ClsClient({
+      region: this.region,
+      secretId: this.credentials.SecretId!,
+      secretKey: this.credentials.SecretKey!,
+      token: this.credentials.Token,
+      debug: false,
+    });
+
+    data.startTime = data.startTime || dayjs(data.endTime).add(-1, 'hour').format(TimeFormat);
+
+    const sql = `SCF_RequestId:${data.reqId} AND SCF_RetryNum:0`;
+    const searchParameters = {
+      logset_id: data.logsetId,
+      topic_ids: data.topicId,
+      start_time: data.startTime as string,
+      end_time: data.endTime,
+      // query_string 必须用 cam 特有的 url 编码方式
+      query_string: sql,
+      limit: 100,
+      sort: 'asc',
+    };
+    const { results = [] } = await clsClient.searchLog(searchParameters);
+    return results;
+  }
 }

src/modules/cls/interface.ts

@@ -32,3 +32,81 @@ export interface ClsDeployInputs
 export interface ClsDeployOutputs extends Partial<ClsDeployInputs> {
   region: RegionType;
 }
+
+export interface StatusSqlMapEnum {
+  success: string;
+  fail: string;
+  retry: string;
+  interrupt: string;
+  timeout: string;
+  exceed: string;
+  codeError: string;
+}
+
+export interface GetSearchSqlOptions {
+  // 函数名称
+  functionName: string;
+  // 命名空间
+  namespace?: string;
+  // 函数版本
+  qualifier?: string;
+  // 开始时间
+  startTime?: number | string;
+  // 结束时间
+  endTime?: number | string;
+  // 请求 ID
+  reqId?: string;
+  // 日志状态
+  status?: keyof StatusSqlMapEnum | '';
+
+  // 查询条数
+  limit?: number;
+}
+
+export type GetLogOptions = Omit<GetSearchSqlOptions, 'startTime'> & {
+  logsetId: string;
+  topicId: string;
+  // 时间间隔，单位秒，默认为 3600s
+  interval?: string | number;
+};
+
+export type GetLogDetailOptions = {
+  logsetId: string;
+  topicId: string;
+  reqId: string;
+  // 开始时间
+  startTime?: string;
+  // 结束时间
+  endTime: string;
+};
+
+export interface LogContent {
+  // 函数名称
+  SCF_FunctionName: string;
+  // 命名空间
+  SCF_Namespace: string;
+  // 开始时间
+  SCF_StartTime: string;
+  // 请求 ID
+  SCF_RequestId: string;
+  // 运行时间
+  SCF_Duration: string;
+  // 别名
+  SCF_Alias: string;
+  // 版本
+  SCF_Qualifier: string;
+  // 日志时间
+  SCF_LogTime: string;
+  // 重试次数
+  SCF_RetryNum: string;
+  // 使用内存
+  SCF_MemUsage: string;
+  // 日志等级
+  SCF_Level: string;
+  // 日志信息
+  SCF_Message: string;
+  // 日志类型
+  SCF_Type: string;
+  // 状态吗
+  SCF_StatusCode: string;
+}

src/modules/cls/utils.ts

@@ -1,6 +1,7 @@
 import { Cls } from '@tencent-sdk/cls';
 import { IndexRule } from '@tencent-sdk/cls/dist/typings';
 import { ApiError } from '../../utils/error';
+import { StatusSqlMapEnum, GetSearchSqlOptions } from './interface';
 
 export async function getLogsetByName(cls: Cls, data: { name: string }) {
   const { logsets = [] } = await cls.getLogsetList();
@@ -240,3 +241,42 @@ export async function deleteClsTrigger(cls: Cls, data: { topic_id: string }) {
   }
   return res;
 }
+
+const StatusSqlMap: StatusSqlMapEnum = {
+  success: 'SCF_StatusCode=200',
+  fail: 'SCF_StatusCode != 200 AND SCF_StatusCode != 202 AND SCF_StatusCode != 499',
+  retry: 'SCF_RetryNum > 0',
+  interrupt: 'SCF_StatusCode = 499',
+  timeout: 'SCF_StatusCode = 433',
+  exceed: 'SCF_StatusCode = 434',
+  codeError: 'SCF_StatusCode = 500',
+};
+
+export function formatWhere({
+  functionName,
+  namespace = 'default',
+  qualifier = '$LATEST',
+  status,
+  startTime,
+  endTime,
+}: Partial<GetSearchSqlOptions>) {
+  let where = `SCF_Namespace='${namespace}' AND SCF_Qualifier='${qualifier}'`;
+  if (startTime && endTime) {
+    where += ` AND (SCF_StartTime between ${startTime} AND ${endTime})`;
+  }
+  if (functionName) {
+    where += ` AND SCF_FunctionName='${functionName}'`;
+  }
+  if (status) {
+    where += ` AND ${StatusSqlMap[status]}'`;
+  }
+
+  return where;
+}
+
+export function getSearchSql(options: GetSearchSqlOptions) {
+  const where = formatWhere(options);
+  const sql = `* | SELECT SCF_RequestId as requestId, SCF_RetryNum as retryNum, MAX(SCF_StartTime) as startTime WHERE ${where} GROUP BY SCF_RequestId, SCF_RetryNum ORDER BY startTime desc`;
+
+  return sql;
+}