From c83d9d61d4b6a8f2982e136edd279f00ab40c076 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sun, 12 May 2024 16:24:13 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=B9=20chore(/config/):=20add=20tsconfi?= =?UTF-8?q?g.json=20&=20linting=20(#2680)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .eslintrc.js | 7 ++ config/translations/anthropic.ts | 110 ++++++++++++++-------------- config/translations/comparisons.ts | 105 ++++++++++++++------------ config/translations/embeddings.ts | 30 ++++---- config/translations/instructions.ts | 9 ++- config/translations/keys.ts | 16 ++-- config/translations/main.ts | 11 ++- config/translations/process.ts | 11 ++- config/translations/scan.ts | 12 ++- config/translations/tsconfig.json | 17 +++++ 10 files changed, 187 insertions(+), 141 deletions(-) create mode 100644 config/translations/tsconfig.json diff --git a/.eslintrc.js b/.eslintrc.js index e85e0d768c..58ee6d20a2 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -132,6 +132,13 @@ module.exports = { }, ], }, + { + files: './config/translations/**/*.ts', + parser: '@typescript-eslint/parser', + parserOptions: { + project: './config/translations/tsconfig.json', + }, + }, { files: ['./packages/data-provider/specs/**/*.ts'], parserOptions: { diff --git a/config/translations/anthropic.ts b/config/translations/anthropic.ts index d7205d5745..f7ab73c442 100644 --- a/config/translations/anthropic.ts +++ b/config/translations/anthropic.ts @@ -1,44 +1,40 @@ import Anthropic from '@anthropic-ai/sdk'; import type * as a from '@anthropic-ai/sdk'; -import { - parseParamFromPrompt, - genTranslationPrompt, -} from '../../api/app/clients/prompts/titlePrompts'; +import { parseParamFromPrompt, genTranslationPrompt } from '~/app/clients/prompts/titlePrompts'; +/** + * Get the initialized Anthropic client. + * @returns {Anthropic} The Anthropic client instance. + */ +export function getClient() { + /** @type {Anthropic.default.RequestOptions} */ + const options = { + apiKey: process.env.ANTHROPIC_API_KEY, + }; - /** - * Get the initialized Anthropic client. - * @returns {Anthropic} The Anthropic client instance. - */ - export function getClient() { - /** @type {Anthropic.default.RequestOptions} */ - const options = { - apiKey: process.env.ANTHROPIC_API_KEY, - }; + return new Anthropic(options); +} - return new Anthropic(options); - } +/** + * This function capitlizes on [Anthropic's function calling training](https://docs.anthropic.com/claude/docs/functions-external-tools). + * + * @param {Object} params - The parameters for the generation. + * @param {string} params.key + * @param {string} params.baselineTranslation + * @param {string} params.translationPrompt + * @param {Array<{ pageContent: string }>} params.context + * + * @returns {Promise} A promise that resolves to the generated conversation title. + * In case of failure, it will return the default title, "New Chat". + */ +export async function translateKeyPhrase({ key, baselineTranslation, translationPrompt, context }) { + let translation: string | undefined; + const model = 'claude-3-sonnet-20240229'; + const prompt = genTranslationPrompt(translationPrompt); + const system = prompt; - /** - * This function capitlizes on [Anthropic's function calling training](https://docs.anthropic.com/claude/docs/functions-external-tools). - * - * @param {Object} params - The parameters for the generation. - * @param {string} params.key - * @param {string} params.baselineTranslation - * @param {string} params.translationPrompt - * @param {Array<{ pageContent: string }>} params.context - * - * @returns {Promise} A promise that resolves to the generated conversation title. - * In case of failure, it will return the default title, "New Chat". - */ - export async function translateKeyPhrase({ key, baselineTranslation, translationPrompt, context }) { - let translation: string | undefined; - const model = 'claude-3-sonnet-20240229'; - const prompt = genTranslationPrompt(translationPrompt); - const system = prompt; - - const translateCompletion = async () => { - const content = `Current key: \`${key}\` + const translateCompletion = async () => { + const content = `Current key: \`${key}\` Baseline translation: ${baselineTranslation} @@ -50,27 +46,27 @@ import { \nsubmit_translation\n\nYour Translation Here\n\n`; - const message: a.Anthropic.MessageParam = { role: 'user', content }; - const requestOptions: a.Anthropic.MessageCreateParamsNonStreaming = { - model, - temperature: 0.3, - max_tokens: 1024, - system, - stop_sequences: ['\n\nHuman:', '\n\nAssistant', ''], - messages: [message], - stream: false, - }; - - try { - const client = getClient(); - const response = await client.messages.create(requestOptions); - const text = response.content[0].text; - translation = parseParamFromPrompt(text, 'translation'); - } catch (e) { - console.error('[AnthropicClient] There was an issue generating the translation', e); - } + const message: a.Anthropic.MessageParam = { role: 'user', content }; + const requestOptions: a.Anthropic.MessageCreateParamsNonStreaming = { + model, + temperature: 0.3, + max_tokens: 1024, + system, + stop_sequences: ['\n\nHuman:', '\n\nAssistant', ''], + messages: [message], + stream: false, }; - await translateCompletion(); - return translation; - } + try { + const client = getClient(); + const response = await client.messages.create(requestOptions); + const text = response.content[0].text; + translation = parseParamFromPrompt(text, 'translation'); + } catch (e) { + console.error('[AnthropicClient] There was an issue generating the translation', e); + } + }; + + await translateCompletion(); + return translation; +} diff --git a/config/translations/comparisons.ts b/config/translations/comparisons.ts index 9a4a4a35fc..90be18c07b 100644 --- a/config/translations/comparisons.ts +++ b/config/translations/comparisons.ts @@ -3,54 +3,65 @@ import path from 'path'; import { exec } from 'child_process'; async function main(baseFilePath: string, languagesDir: string) { - const { default: baseLanguage } = await import(path.resolve(baseFilePath)); - const files = fs.readdirSync(languagesDir); + const { default: baseLanguage } = await import(path.resolve(baseFilePath)); + const files = fs.readdirSync(languagesDir); - for (let file of files) { - const ext = path.extname(file); - if (ext !== '.ts' && ext !== '.tsx') continue; // Only process TypeScript files - - const filePath = path.resolve(languagesDir, file); - if (filePath === baseFilePath) continue; // Skip the base language file - - const { default: otherLanguage } = await import(filePath); - let comparisons = {}; - - for (let key in otherLanguage) { - if (otherLanguage.hasOwnProperty(key) && baseLanguage.hasOwnProperty(key)) { - comparisons[key] = { - english: baseLanguage[key], - translated: otherLanguage[key] - }; - } - } - - let fileContent = fs.readFileSync(filePath, 'utf8'); - const comparisonsObjRegex = /export const comparisons = {[\s\S]*?};/gm; - const hasComparisons = comparisonsObjRegex.test(fileContent); - const comparisonsExport = `\nexport const comparisons = ${JSON.stringify(comparisons, null, 2)};\n`; - - if (hasComparisons) { - fileContent = fileContent.replace(comparisonsObjRegex, comparisonsExport); - } else { - fileContent = fileContent.trim() + comparisonsExport; - } - - fs.writeFileSync(filePath, fileContent); // Write updated content back to file + for (const file of files) { + const ext = path.extname(file); + if (ext !== '.ts' && ext !== '.tsx') { + continue; } - // Execute ESLint with the --fix option on the entire directory - exec(`bunx eslint "${languagesDir}" --fix`, (error, stdout, stderr) => { - if (error) { - console.error('Error executing ESLint:', error); - return; - } - if (stderr) { - console.error('ESLint stderr:', stderr); - return; - } - console.log('ESLint stdout:', stdout); - }); + const filePath = path.resolve(languagesDir, file); + if (filePath === baseFilePath) { + continue; + } + + const { default: otherLanguage } = await import(filePath); + const comparisons = {}; + + for (const key in otherLanguage) { + if ( + Object.prototype.hasOwnProperty.call(otherLanguage, key) && + Object.prototype.hasOwnProperty.call(baseLanguage, key) + ) { + comparisons[key] = { + english: baseLanguage[key], + translated: otherLanguage[key], + }; + } + } + + let fileContent = fs.readFileSync(filePath, 'utf8'); + const comparisonsObjRegex = /export const comparisons = {[\s\S]*?};/gm; + const hasComparisons = comparisonsObjRegex.test(fileContent); + const comparisonsExport = `\nexport const comparisons = ${JSON.stringify( + comparisons, + null, + 2, + )};\n`; + + if (hasComparisons) { + fileContent = fileContent.replace(comparisonsObjRegex, comparisonsExport); + } else { + fileContent = fileContent.trim() + comparisonsExport; + } + + fs.writeFileSync(filePath, fileContent); + } + + // Execute ESLint with the --fix option on the entire directory + exec(`bunx eslint "${languagesDir}" --fix`, (error, stdout, stderr) => { + if (error) { + console.error('Error executing ESLint:', error); + return; + } + if (stderr) { + console.error('ESLint stderr:', stderr); + return; + } + console.log('ESLint stdout:', stdout); + }); } const languagesDir = './client/src/localization/languages'; @@ -58,15 +69,13 @@ const baseFilePath = path.resolve(languagesDir, 'Eng.ts'); main(baseFilePath, languagesDir).catch(console.error); - // const prompt = ` // Write a prompt that is mindful of the nuances in the language with respect to its English counterpart, which serves as the baseline for translations. Here are the comparisons between the language translations and their English counterparts: // ${comparisons} - // Please consider the above comparisons to enhance understanding and guide improvements in translations. Provide insights or suggestions that could help refine the translation process, focusing on cultural and contextual relevance. // Please craft a prompt that can be used to better inform future translations to this language. Write this prompt in the translated language, with all its nuances detected, not in the English. -// `; \ No newline at end of file +// `; diff --git a/config/translations/embeddings.ts b/config/translations/embeddings.ts index 4d56e0fe71..8d0bf48f24 100644 --- a/config/translations/embeddings.ts +++ b/config/translations/embeddings.ts @@ -2,34 +2,34 @@ import dotenv from 'dotenv'; dotenv.config({ path: './', }); -import { OpenAIEmbeddings } from "@langchain/openai"; -import { HNSWLib } from "@langchain/community/vectorstores/hnswlib"; -import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"; -import * as fs from "fs"; -import * as path from "path"; +import { OpenAIEmbeddings } from '@langchain/openai'; +import { HNSWLib } from '@langchain/community/vectorstores/hnswlib'; +import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'; +import * as fs from 'fs'; +import * as path from 'path'; export const storeEmbeddings = async (modulePath: string) => { try { - const text = fs.readFileSync(modulePath, "utf8"); + const text = fs.readFileSync(modulePath, 'utf8'); const textSplitter = new RecursiveCharacterTextSplitter({ chunkSize: 600 }); const docs = await textSplitter.createDocuments([text]); const vectorStore = await HNSWLib.fromDocuments(docs, new OpenAIEmbeddings()); const directory = `./config/translations/stores/${path.basename(modulePath)}`; if (!fs.existsSync(directory)) { - fs.mkdirSync(directory, { recursive: true }); - console.log(`Directory created: ${directory}`); + fs.mkdirSync(directory, { recursive: true }); + console.log(`Directory created: ${directory}`); } else { - console.log(`Directory already exists: ${directory}`); - return; + console.log(`Directory already exists: ${directory}`); + return; } await vectorStore.save(directory); } catch (error) { - console.error(`Error storing embeddings`); - console.error(error); + console.error('Error storing embeddings'); + console.error(error); } -} +}; export const loadEmbeddings = async (modulePath: string) => { try { @@ -37,7 +37,7 @@ export const loadEmbeddings = async (modulePath: string) => { const loadedVectorStore = await HNSWLib.load(directory, new OpenAIEmbeddings()); return loadedVectorStore; } catch (error) { - console.error(`Error loading embeddings`); + console.error('Error loading embeddings'); console.error(error); } -} \ No newline at end of file +}; diff --git a/config/translations/instructions.ts b/config/translations/instructions.ts index e0f575d8ff..205d9da630 100644 --- a/config/translations/instructions.ts +++ b/config/translations/instructions.ts @@ -5,11 +5,13 @@ const baseDirPath = './client/src/localization/languages'; const promptsDirPath = './client/src/localization/prompts/instructions'; async function ensureDirectoryExists(directory: string) { - return fs.promises.access(directory).catch(() => fs.promises.mkdir(directory, { recursive: true })); + return fs.promises + .access(directory) + .catch(() => fs.promises.mkdir(directory, { recursive: true })); } // Helper function to generate Markdown from an object, recursively if needed -function generateMarkdownFromObject(obj: any, depth: number = 0): string { +function generateMarkdownFromObject(obj: object, depth = 0): string { if (typeof obj !== 'object' || obj === null) { return String(obj); } @@ -61,7 +63,8 @@ async function createPromptsForTranslations() { const files = await fs.promises.readdir(baseDirPath); for (const file of files) { - if (!file.includes('Eng.ts')) { // Ensure English or base file is excluded + if (!file.includes('Eng.ts')) { + // Ensure English or base file is excluded const filePath = path.join(baseDirPath, file); const promptContent = await generatePromptForFile(filePath, file); const outputFilePath = path.join(promptsDirPath, `${path.basename(file, '.ts')}.md`); diff --git a/config/translations/keys.ts b/config/translations/keys.ts index a923ce3917..362c7b137b 100644 --- a/config/translations/keys.ts +++ b/config/translations/keys.ts @@ -8,21 +8,25 @@ async function readKeysFromFile(filePath: string): Promise { } async function compareKeys(baseKeys: string[], keysFromOtherFile: string[]): Promise { - const missingKeys = baseKeys.filter(key => !keysFromOtherFile.includes(key)); + const missingKeys = baseKeys.filter((key) => !keysFromOtherFile.includes(key)); return missingKeys; } async function main(baseFilePath: string, languagesDir: string) { const baseKeys = await readKeysFromFile(baseFilePath); - + const files = fs.readdirSync(languagesDir); for (const file of files) { const ext = path.extname(file); - if (ext !== '.ts' && ext !== '.tsx') continue; // Ensure it's a TypeScript file - + if (ext !== '.ts' && ext !== '.tsx') { + continue; + } // Ensure it's a TypeScript file + const compareFilePath = path.resolve(languagesDir, file); - if (compareFilePath === baseFilePath) continue; // Skip the base file - + if (compareFilePath === baseFilePath) { + continue; + } // Skip the base file + try { const keysFromOtherFile = await readKeysFromFile(compareFilePath); const missingKeys = await compareKeys(baseKeys, keysFromOtherFile); diff --git a/config/translations/main.ts b/config/translations/main.ts index 747054ecad..09be7edaf4 100644 --- a/config/translations/main.ts +++ b/config/translations/main.ts @@ -16,11 +16,11 @@ export default async function main(baseFilePath: string, compareFilePath: string const compareModule = await import(compareFilePath); const compareKeys = Object.keys(compareModule.default); - const missingKeys = baseKeys.filter(key => !compareKeys.includes(key)); + const missingKeys = baseKeys.filter((key) => !compareKeys.includes(key)); if (missingKeys.length > 0) { const keyTranslations = {}; for (const key of missingKeys) { - const baselineTranslation = baseModule.default[key] || "No baseline translation available"; + const baselineTranslation = baseModule.default[key] || 'No baseline translation available'; const result = await processMissingKey({ key, baselineTranslation, @@ -31,8 +31,11 @@ export default async function main(baseFilePath: string, compareFilePath: string } const outputDir = path.dirname(compareFilePath); - const outputFileName = `${path.basename(compareFilePath, path.extname(compareFilePath))}_missing_keys.json`; + const outputFileName = `${path.basename( + compareFilePath, + path.extname(compareFilePath), + )}_missing_keys.json`; const outputFilePath = path.join(outputDir, outputFileName); fs.writeFileSync(outputFilePath, JSON.stringify(keyTranslations, null, 2)); } -} \ No newline at end of file +} diff --git a/config/translations/process.ts b/config/translations/process.ts index c87f794ae0..7494dbbf60 100644 --- a/config/translations/process.ts +++ b/config/translations/process.ts @@ -20,7 +20,7 @@ export async function processLanguageModule(moduleName: string, modulePath: stri await storeEmbeddings(modulePath); vectorStoreMap[moduleName] = await loadEmbeddings(modulePath); const baseKeys = Object.keys((await import(modulePath)).default); - console.log(`Keys in module: ${moduleName}:`, baseKeys.length) + console.log(`Keys in module: ${moduleName}:`, baseKeys.length); missingKeyMap[moduleName] = 0; return prompt; } @@ -30,8 +30,11 @@ export async function processMissingKey({ baselineTranslation, moduleName, translationPrompt, -} : { - key: string, baselineTranslation: string, moduleName: string, translationPrompt: string +}: { + key: string; + baselineTranslation: string; + moduleName: string; + translationPrompt: string; }) { missingKeyMap[moduleName]++; const vectorStore = vectorStoreMap[moduleName]; @@ -42,6 +45,6 @@ export async function processMissingKey({ translationPrompt, context, }); - console.log(`"${key}": "${translation}",\n`) + console.log(`"${key}": "${translation}",\n`); return translation; } diff --git a/config/translations/scan.ts b/config/translations/scan.ts index f046212994..04a4e9b13e 100644 --- a/config/translations/scan.ts +++ b/config/translations/scan.ts @@ -6,11 +6,15 @@ async function scanDirectory(baseFilePath: string, languagesDir: string) { const files = fs.readdirSync(languagesDir); for (const file of files) { const ext = path.extname(file); - if (ext !== '.ts' && ext !== '.tsx') continue; - + if (ext !== '.ts' && ext !== '.tsx') { + continue; + } + const compareFilePath = path.resolve(languagesDir, file); - if (compareFilePath === baseFilePath) continue; - + if (compareFilePath === baseFilePath) { + continue; + } + await main(baseFilePath, compareFilePath); } } diff --git a/config/translations/tsconfig.json b/config/translations/tsconfig.json new file mode 100644 index 0000000000..3d3dd26ca3 --- /dev/null +++ b/config/translations/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "noImplicitAny": false, + "baseUrl": ".", + "outDir": "./dist", + "strict": true, + "moduleResolution": "node", + "esModuleInterop": true, + "target": "es6", + "module": "esnext", + "lib": ["dom", "es6"], + "paths": { + "~/*": ["../../api/*"] + } + }, + "include": ["*.ts"] +}