feat: split text into chunks if too large

Author: xiaoyu2er

packages/translate/src/chunk.ts

@@ -0,0 +1,87 @@
+// Constants for token estimation
+export const CHAR_TO_TOKEN_RATIO = 0.3; // 1 English character ≈ 0.3 token
+export const CHAR_TO_TOKEN_RATIO_ZH = 0.5; // 1 Chinese character ≈ 0.5 token
+export const MAX_INPUT_TOKENS = 64 * 1024; // DeepSeek's 64K context length
+export const MAX_OUTPUT_TOKENS = 8 * 1024; // DeepSeek's 8K max output
+
+// Chunk size constants (in estimated tokens)
+export const MAX_CHUNK_SIZE_TOKENS = 16 * 1024; // Use smaller chunks for better translation quality
+
+// Helper function to estimate tokens based on content
+export function estimateTokens(content: string): number {
+  // we use Chinese ratio because Chinese is concise
+  return Math.ceil(content.length * CHAR_TO_TOKEN_RATIO_ZH);
+}
+
+export function needsChunking(content: string): boolean {
+  return estimateTokens(content) > MAX_OUTPUT_TOKENS;
+}
+
+// Split text into chunks that respect markdown structure and heading hierarchy
+export function splitIntoChunks(content: string): string[] {
+  // Define a regex pattern for markdown headings (## Heading)
+  const headingPattern = /^(#{2,}) /gm;
+
+  // Split content by headings while keeping the headings with the content that follows
+  const sections: string[] = [];
+  let lastIndex = 0;
+
+  // Find all heading matches
+  const contentWithExtraLineBreak = `${content}\n\n`;
+  const matches = Array.from(
+    contentWithExtraLineBreak.matchAll(headingPattern),
+  );
+
+  if (matches.length > 0) {
+    sections.push(content.substring(0, matches[0].index));
+  } else {
+    sections.push(content);
+  }
+
+  // Process each heading match
+  for (let i = 0; i < matches.length; i++) {
+    const match = matches[i];
+    const currentIndex = match.index !== undefined ? match.index : 0;
+
+    // If this isn't the first heading, add the section between last heading and this one
+    if (i > 0 && lastIndex < currentIndex) {
+      sections.push(content.substring(lastIndex, currentIndex));
+    }
+
+    // Update lastIndex for next iteration
+    lastIndex = currentIndex;
+
+    // If this is the last heading, add the section from this heading to the end
+    if (i === matches.length - 1) {
+      sections.push(content.substring(lastIndex));
+    }
+  }
+
+  // Further process sections to ensure they don't exceed token limits
+  const chunks = [];
+  let currentChunk = '';
+  let currentTokens = 0;
+
+  for (const section of sections) {
+    const sectionTokens = estimateTokens(section);
+
+    if (currentTokens + sectionTokens > MAX_OUTPUT_TOKENS) {
+      // If adding this section would exceed the limit, start a new chunk
+      chunks.push(currentChunk);
+      currentChunk = section;
+      currentTokens = sectionTokens;
+    } else {
+      currentChunk += section;
+      currentTokens += sectionTokens;
+    }
+  }
+
+  // Add any remaining content as the last chunk
+  if (currentChunk) {
+    chunks.push(currentChunk);
+  }
+
+  // console.log('chunks-tokens', chunks.map((chunk) => estimateTokens(chunk)));
+
+  return chunks;
+}

packages/translate/src/main.ts

@@ -3,16 +3,11 @@ import * as path from 'node:path';
 import micromatch from 'micromatch';
 import { executeInBatches } from './batch';
 import { logger } from './logger';
-import { $translateConfig } from './openai';
 import type { MainConfig } from './types';
 import {
-  copyDoc,
-  extractPathToLabelMap,
   findDocFiles,
   getDocUpdateStatus,
-  getTranslatedConfig,
   normalizePatterns,
-  shouldTranslateConfig,
   translateDoc,
 } from './utils';
 
@@ -150,7 +145,7 @@ export async function main({
 
       // Check if this path should be copied without translation
 
-      const [shouldUpdate, reason] = await getDocUpdateStatus({
+      const { shouldUpdate, reason, chunks } = await getDocUpdateStatus({
         sourcePath,
         targetPath,
       });
@@ -159,6 +154,7 @@ export async function main({
         Source: sourcePath,
         Target: targetPath,
         'Should update?': shouldUpdate ? '✅ Yes' : '❌ No',
+        Chunks: chunks,
         Reason: reason,
       });
 

packages/translate/src/openai.ts

@@ -1,4 +1,11 @@
 import OpenAI from 'openai';
+import type { ChatCompletionMessageParam } from 'openai/resources.mjs';
+import {
+  MAX_OUTPUT_TOKENS,
+  estimateTokens,
+  needsChunking,
+  splitIntoChunks,
+} from './chunk';
 import { logger } from './logger';
 
 interface LangConfig {
@@ -33,17 +40,17 @@ export const model = 'deepseek-chat';
 export const systemPrompt =
   'You are a professional technical translator specializing in software documentation. You are particularly skilled at translating React, web development, and programming terminology, keeping the translations consistent and readable.';
 
-// Improved $translateDocument function with language-specific prompts
-export async function $translateDocument({
-  content,
-  langConfig,
-  context = '',
-}: TranslateDocumentParams): Promise<string> {
+// Helper function to translate a single chunk
+async function translateChunk(
+  chunk: string,
+  langConfig: LangConfig,
+  context: string,
+): Promise<string> {
   if (!openai) {
     throw new Error('OPENAI_API_KEY is not set.');
   }
 
-  const textLength = content.length;
+  const textLength = chunk.length;
   const prompt = `
 Translate the following documentation from English to ${langConfig.name}.
 The document being translated is of MDX file type, which means there are components in the text. Please ensure component closing tags are not missing, for example:
@@ -60,33 +67,101 @@ HERE IS THE TEXT TO TRANSLATE:
 `;
 
   logger.debug(
-    `Sending translation request, text length: ${textLength} characters, prompt totoal length: ${
+    `Sending chunk translation request, text length: ${textLength} characters, prompt total length: ${
       prompt.length + textLength
-    } characters`,
+    } characters, estimated tokens: ${estimateTokens(prompt + chunk)} tokens`,
   );
 
-  // logger.debug(`Prompt:\n${prompt}`);
+  const messages: Array<ChatCompletionMessageParam> = [
+    {
+      role: 'system',
+      content: systemPrompt,
+    },
+    {
+      role: 'user',
+      content: prompt,
+    },
+    {
+      role: 'user',
+      content: chunk,
+    },
+  ];
 
   const response = await openai.chat.completions.create({
     model: model,
-    max_completion_tokens: 8 * 1024,
-    max_tokens: 8 * 1024,
-    messages: [
-      {
-        role: 'system',
-        content: systemPrompt,
-      },
-      {
-        role: 'user',
-        content: prompt + content,
-      },
-    ],
+    max_completion_tokens: MAX_OUTPUT_TOKENS,
+    max_tokens: MAX_OUTPUT_TOKENS,
+    messages: messages,
   });
 
-  const translatedContent = response.choices?.[0]?.message?.content;
-  if (!translatedContent) {
+  logger.debug(`response.usage:\n${JSON.stringify(response.usage, null, 2)}`);
+  const message = response.choices?.[0]?.message;
+
+  if (!message?.content) {
     throw new Error('Failed to get translation response');
   }
-  logger.debug(`Response:\n${response}`);
-  return translatedContent.trim();
+  // Return the translated content and the message to add to history
+  return message.content;
+}
+
+// New implementation for translating large documents using multi-round chat
+export async function $translateDocument({
+  content,
+  langConfig,
+  context = '',
+}: TranslateDocumentParams): Promise<string> {
+  if (!openai) {
+    throw new Error('OPENAI_API_KEY is not set.');
+  }
+
+  const textLength = content.length;
+  logger.debug(
+    `Starting translation of document with length: ${textLength} characters, estimated tokens: ${estimateTokens(content)} tokens`,
+  );
+
+  // For small documents, use the direct approach
+  if (!needsChunking(content)) {
+    return await translateChunk(content, langConfig, context);
+  }
+
+  logger.debug(
+    'Document is large, splitting into chunks for multi-round translation',
+  );
+  const chunks = splitIntoChunks(content);
+  logger.debug(`Split document into ${chunks.length} chunks`);
+
+  let translatedContent = '';
+  for (let i = 0; i < chunks.length; i++) {
+    const chunk = chunks[i];
+    logger.debug(`Translating chunk ${i + 1} of ${chunks.length}`);
+
+    // Translate the chunk
+    const translatedChunk = await translateChunk(chunk, langConfig, context);
+
+    // Add to the complete translated content
+    if (chunk[0] === '\n' && translatedChunk[0] !== '\n') {
+      translatedContent += '\n';
+    }
+    if (chunk[1] === '\n' && translatedChunk[1] !== '\n') {
+      translatedContent += '\n';
+    }
+
+    translatedContent += translatedChunk;
+
+    if (
+      chunk[chunk.length - 1] === '\n' &&
+      translatedChunk[translatedChunk.length - 1] !== '\n'
+    ) {
+      translatedContent += '\n';
+    }
+    if (
+      chunk[chunk.length - 2] === '\n' &&
+      translatedChunk[translatedChunk.length - 2] !== '\n'
+    ) {
+      translatedContent += '\n';
+    }
+  }
+
+  logger.debug(`Completed translation of all ${chunks.length} chunks`);
+  return translatedContent;
 }

packages/translate/src/utils.ts

@@ -3,6 +3,7 @@ import * as fs$ from 'node:fs/promises';
 import path from 'node:path';
 import { glob } from 'glob';
 import matter from 'gray-matter';
+import { needsChunking, splitIntoChunks } from './chunk';
 import { logger } from './logger';
 import { $translateDocument } from './openai';
 
@@ -57,17 +58,24 @@ export function getLastModifiedTimeFromGit(filePath: string): Date {
 export async function getDocUpdateStatus({
   sourcePath,
   targetPath,
-}: CheckFileUpdateParams): Promise<[boolean, string]> {
+}: CheckFileUpdateParams): Promise<{
+  shouldUpdate: boolean;
+  chunks: 'N/A' | number;
+  reason: string;
+}> {
   try {
     await fs$.access(sourcePath);
   } catch (error) {
     logger.error(
       `Source file not found: ${sourcePath}, don't need updating, consider removing it`,
     );
-    return [false, 'Source not found'];
+    return { shouldUpdate: false, chunks: 'N/A', reason: 'Source not found' };
   }
 
   const sourceContent = await fs$.readFile(sourcePath, 'utf8');
+  const chunks = needsChunking(sourceContent)
+    ? splitIntoChunks(sourceContent).length
+    : 1;
   const sourceParsed = matter(sourceContent);
 
   let sourceLastModifiedDate = getLastModifiedTimeFromGit(sourcePath);
@@ -78,7 +86,11 @@ export async function getDocUpdateStatus({
       logger.error(
         `Referenced file not found: ${sourceParsed.data.ref}, don't need updating, consider REMOVING it`,
       );
-      return [false, 'Referenced file not found'];
+      return {
+        shouldUpdate: false,
+        reason: 'Referenced file not found',
+        chunks,
+      };
     }
 
     const refLastModifiedDate = getLastModifiedTimeFromGit(
@@ -93,7 +105,7 @@ export async function getDocUpdateStatus({
     await fs$.access(targetPath);
   } catch (error) {
     logger.debug(`Target file not found: ${targetPath}, needs updating`);
-    return [true, 'Target not found.'];
+    return { shouldUpdate: true, reason: 'Target not found.', chunks };
   }
 
   // Read target file and parse frontmatter
@@ -112,16 +124,28 @@ export async function getDocUpdateStatus({
       logger.debug(
         `Source file ${sourcePath} has been updated since last translation, needs updating`,
       );
-      return [true, 'Source has been modified. '];
+      return {
+        shouldUpdate: true,
+        reason: 'Source has been modified. ',
+        chunks,
+      };
     }
-    return [false, 'Source has not been modified.'];
+    return {
+      shouldUpdate: false,
+      reason: 'Source has not been modified.',
+      chunks,
+    };
   }
 
   // If there's no source-updated-at in target, it needs to be updated
   logger.debug(
     `Target file ${targetPath} has no source-updated-at metadata, needs updating`,
   );
-  return [true, 'Target no source-updated-at metadata, needs updating.'];
+  return {
+    shouldUpdate: true,
+    reason: 'Target no source-updated-at metadata, needs updating.',
+    chunks,
+  };
 }
 
 // New helper function to extract context from overview files
@@ -191,19 +215,12 @@ export async function translateDoc({
   const sourceUpdatedAt = getLastModifiedTimeFromGit(sourcePath).toISOString();
   const translationUpdatedAt = new Date().toISOString();
 
-  const parsed = matter(translatedContent);
-
-  // Create frontmatter data object
-  const frontmatterData = {
-    'source-updated-at': sourceUpdatedAt,
-    'translation-updated-at': translationUpdatedAt,
-    ...parsed.data,
-  };
-
-  logger.debug(
-    `Writing translated content, ${JSON.stringify(frontmatterData)}`,
+  const newContent = translatedContent.replace(
+    '---',
+    `---
+source-updated-at: ${sourceUpdatedAt}
+translation-updated-at: ${translationUpdatedAt}`,
   );
-  const newContent = matter.stringify(parsed.content, frontmatterData);
 
   logger.debug(`Writing translated content to ${targetPath}`);
   await fs$.writeFile(targetPath, newContent, 'utf8');