Pull request: AGDNS-4161-imp-translation-scripts

Squashed commit of the following:

commit 8a79a364cf0614a3c7ccef6efcfaae546a6087c2
Merge: 70bb94848 5c4af121d
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Mon Jun 29 09:17:02 2026 +0700

    Merge remote-tracking branch 'origin/master' into AGDNS-4161-imp-translation-scripts

commit 70bb9484864405bc1776af1138578d90bdea76e1
Merge: aa62ca71f 07d8d77a5
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri Jun 26 08:50:13 2026 +0700

    Merge remote-tracking branch 'origin/master' into AGDNS-4161-imp-translation-scripts

commit aa62ca71f859f359eb5db1d2c354afc07b4e1369
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri Jun 26 08:49:55 2026 +0700

    scripts: imp docs

commit 4195a5dbbc73a2bef5548aa0ce54709f566e1608
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Thu Jun 25 10:59:20 2026 +0700

    scripts: fix translations

commit be40c845cd86e21f3a4a08982ce6d8801149e643
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Thu Jun 25 10:55:55 2026 +0700

    scripts: imp code

commit 9fa78efe7115be8496cf89d40dbcc8e7a4220cbc
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Thu Jun 25 10:16:08 2026 +0700

    scripts: imp code

commit ad2cbc31285dba5a870098945cded8a22ee1418c
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Wed Jun 24 10:46:53 2026 +0700

    scripts: imp code

.twosky.json

@@ -42,6 +42,10 @@
       "zh-cn": "简体中文",
       "zh-hk": "繁體中文（香港）",
       "zh-tw": "正體中文（台灣）"
+    },
+    "metadata": {
+      "locales_dir": "./client/src/__locales",
+      "sources_dir": "./client/src"
     }
   },
   {
@@ -87,6 +91,10 @@
       "zh-cn": "简体中文",
       "zh-hk": "繁體中文（香港）",
       "zh-tw": "正體中文（台灣）"
+    },
+    "metadata": {
+      "locales_dir": "./client/src/__locales-services",
+      "sources_dir": "./client/src"
     }
   }
 ]

scripts/README.md

@@ -206,7 +206,7 @@ Optional environment:
 
 - `go run ./scripts/translations auto-add`: add locales with additions to the git and restore locales with deletions.
 
-After the download you’ll find the output locales in the `client/src/__locales/` directory.
+After the download you’ll find the output locales in the configured directory.
 
 Optional environment:
 
@@ -216,9 +216,7 @@ Optional environment:
 
 - `TWOSKY_URI`: set an alternative URL for `download` or `upload`.
 
-- `TWOSKY_PROJECT_ID`: set an alternative project ID for `download` or `upload`.
-
-    Deprectated: This environment variable should not be used since the script began supporting multiple configurations.
+- `TWOSKY_PROJECT_ID`: set an alternative project ID. The default value is `home`.
 
 ## `companiesdb/`: Whotracks.me database converter
 

scripts/translations/download.go

@@ -49,15 +49,15 @@ func (c *twoskyClient) download(ctx context.Context, l *slog.Logger) {
 		wg.Go(dw.run)
 	}
 
-	for _, baseFile := range c.localizableFiles {
-		dir, file := filepath.Split(baseFile)
+	for _, mask := range c.localizableFiles {
+		_, file := filepath.Split(mask)
 
 		for _, lang := range c.langs {
 			uri := translationURL(downloadURI, file, c.projectID, lang)
 
 			reqCh <- downloadRequest{
 				uri: uri,
-				dir: dir,
+				dir: c.localesDir,
 			}
 		}
 	}

scripts/translations/main.go

@@ -26,11 +26,10 @@ import (
 
 // TODO(e.burkov):  Remove the default as they should be set by configuration.
 const (
-	twoskyConfFile  = "./.twosky.json"
-	localesDirHome  = "./client/src/__locales"
-	defaultBaseFile = "en.json"
-	srcDir          = "./client/src"
-	twoskyURI       = "https://twosky.int.agrd.dev/api/v1"
+	defaultBaseFile  = "en.json"
+	defaultProjectID = "home"
+	twoskyConfFile   = "./.twosky.json"
+	twoskyURI        = "https://twosky.int.agrd.dev/api/v1"
 
 	readLimit     = 1 * datasize.MB
 	uploadTimeout = 20 * time.Second
@@ -76,27 +75,20 @@ func main() {
 		usage("")
 	}
 
-	homeConf, servicesConf, err := readTwoskyConfig()
-	errors.Check(err)
-
-	var cli *twoskyClient
+	conf := errors.Must(newTwoskyConfig())
+	cli := errors.Must(newTwoskyClient(conf))
 
 	switch os.Args[1] {
 	case "summary":
-		errors.Check(summary(homeConf.Languages))
+		errors.Check(cli.summary())
 	case "download":
-		cli = errors.Must(newTwoskyClient(homeConf))
-		cli.download(ctx, l)
-
-		cli = errors.Must(newTwoskyClient(servicesConf))
 		cli.download(ctx, l)
 	case "unused":
-		errors.Check(unused(ctx, l, homeConf.LocalizableFiles[0]))
+		errors.Check(cli.unused(ctx, l))
 	case "upload":
-		cli = errors.Must(newTwoskyClient(homeConf))
 		errors.Check(cli.upload())
 	case "auto-add":
-		errors.Check(autoAdd(ctx, l, homeConf.LocalizableFiles[0]))
+		errors.Check(cli.autoAdd(ctx, l))
 	default:
 		usage("unknown command")
 	}
@@ -151,10 +143,10 @@ func validateLanguageStr(str string, all languages) (langs []langCode, err error
 	return langs, nil
 }
 
-// readLocales reads file with name fn and returns a map, where key is text
-// label and value is localization.
-func readLocales(fn string) (loc locales, err error) {
-	b, err := os.ReadFile(fn)
+// readLocales reads file by name and returns a map, where the key is a text
+// label and the value is a localization.
+func readLocales(name string) (loc locales, err error) {
+	b, err := os.ReadFile(name)
 	if err != nil {
 		// Don't wrap the error since it's informative enough as is.
 		return nil, err
@@ -163,46 +155,44 @@ func readLocales(fn string) (loc locales, err error) {
 	loc = make(locales)
 	err = json.Unmarshal(b, &loc)
 	if err != nil {
-		err = fmt.Errorf("unmarshalling %q: %w", fn, err)
-
-		return nil, err
+		return nil, fmt.Errorf("unmarshalling %q: %w", name, err)
 	}
 
 	return loc, nil
 }
 
-// summary prints summary for translations.
-//
-// TODO(e.burkov):  Consider making it a method of [twoskyClient] and
-// calculating summary for all configurations.
-func summary(langs languages) (err error) {
-	basePath := filepath.Join(localesDirHome, defaultBaseFile)
+// summary prints a summary for translations.
+func (c *twoskyClient) summary() (err error) {
+	defer func() { err = errors.Annotate(err, "summary: %w") }()
+
+	fmt.Printf("Summary for %s:\n\n", c.projectID)
+
+	basePath := filepath.Join(c.localesDir, defaultBaseFile)
 	baseLoc, err := readLocales(basePath)
 	if err != nil {
-		return fmt.Errorf("summary: %w", err)
+		return fmt.Errorf("reading base locales: %w", err)
 	}
 
 	size := float64(len(baseLoc))
 
-	for _, lang := range slices.Sorted(maps.Keys(langs)) {
-		name := filepath.Join(localesDirHome, string(lang)+".json")
+	for _, lang := range c.langs {
+		name := filepath.Join(c.localesDir, string(lang)+".json")
 		if name == basePath {
+			// Skip base lang.
 			continue
 		}
 
 		var loc locales
 		loc, err = readLocales(name)
 		if err != nil {
-			return fmt.Errorf("summary: reading locales: %w", err)
+			return fmt.Errorf("reading locales for %s: %w", lang, err)
 		}
 
 		f := float64(len(loc)) * 100 / size
 
 		blocker := ""
-
 		// N is small enough to not raise performance questions.
-		ok := slices.Contains(blockerLangCodes, lang)
-		if ok {
+		if slices.Contains(blockerLangCodes, lang) {
 			blocker = " (blocker)"
 		}
 
@@ -212,32 +202,39 @@ func summary(langs languages) (err error) {
 	return nil
 }
 
-// unused prints unused text labels.
-//
-// TODO(e.burkov):  Consider making it a method of [twoskyClient] and searching
-// unused strings for all configurations.
-func unused(ctx context.Context, l *slog.Logger, basePath string) (err error) {
+// unused prints unused text labels.  l must not be nil.
+func (c *twoskyClient) unused(ctx context.Context, l *slog.Logger) (err error) {
 	defer func() { err = errors.Annotate(err, "unused: %w") }()
 
-	baseLoc, err := readLocales(basePath)
+	fmt.Printf("Unused for %s:\n\n", c.projectID)
+	fmt.Println()
+
+	basePath := filepath.Join(c.localesDir, defaultBaseFile)
+	baseLocales, err := readLocales(basePath)
 	if err != nil {
-		return err
+		return fmt.Errorf("reading locales: %w", err)
 	}
 
-	locDir := filepath.Clean(localesDirHome)
-	js, err := findJS(ctx, l, locDir)
+	srcFiles, err := c.findSourceFiles(ctx, l)
 	if err != nil {
-		return err
+		return fmt.Errorf("collecting js files: %w", err)
 	}
 
-	return findUnused(js, baseLoc)
+	err = findUnused(srcFiles, baseLocales)
+	if err != nil {
+		return fmt.Errorf("finding unused files: %w", err)
+	}
+
+	return nil
 }
 
-// findJS returns list of JavaScript and JSON files or error.
-func findJS(ctx context.Context, l *slog.Logger, locDir string) (fileNames []string, err error) {
-	walkFn := func(name string, _ os.FileInfo, err error) error {
-		if err != nil {
-			l.WarnContext(ctx, "accessing a path", slogutil.KeyError, err)
+// findSourceFiles returns list of source files.  l must not be nil.
+func (c *twoskyClient) findSourceFiles(ctx context.Context, l *slog.Logger) (fileNames []string, err error) {
+	locDir := filepath.Clean(c.localesDir)
+
+	walkFn := func(name string, _ os.FileInfo, pErr error) (err error) {
+		if pErr != nil {
+			l.WarnContext(ctx, "accessing a path", slogutil.KeyError, pErr)
 
 			return nil
 		}
@@ -247,16 +244,16 @@ func findJS(ctx context.Context, l *slog.Logger, locDir string) (fileNames []str
 		}
 
 		ext := filepath.Ext(name)
-		if ext == ".js" || ext == ".json" {
+		if ext == ".js" || ext == ".json" || ext == ".ts" || ext == ".tsx" {
 			fileNames = append(fileNames, name)
 		}
 
 		return nil
 	}
 
-	err = filepath.Walk(srcDir, walkFn)
+	err = filepath.Walk(c.sourcesDir, walkFn)
 	if err != nil {
-		return nil, fmt.Errorf("filepath walking %q: %w", srcDir, err)
+		return nil, fmt.Errorf("walking %q: %w", c.sourcesDir, err)
 	}
 
 	return fileNames, nil
@@ -278,7 +275,7 @@ func findUnused(fileNames []string, loc locales) (err error) {
 		var buf []byte
 		buf, err = os.ReadFile(fn)
 		if err != nil {
-			return fmt.Errorf("finding unused: %w", err)
+			return fmt.Errorf("reading file: %w", err)
 		}
 
 		for k := range loc {
@@ -296,18 +293,19 @@ func findUnused(fileNames []string, loc locales) (err error) {
 }
 
 // autoAdd adds locales with additions to the git and restores locales with
-// deletions.
-func autoAdd(ctx context.Context, l *slog.Logger, basePath string) (err error) {
+// deletions.  l must not be nil.
+func (c *twoskyClient) autoAdd(ctx context.Context, l *slog.Logger) (err error) {
 	defer func() { err = errors.Annotate(err, "auto add: %w") }()
 
 	cmdCons := executil.SystemCommandConstructor{}
 
-	adds, dels, err := changedLocales(ctx, l, cmdCons)
+	adds, dels, err := c.changedLocales(ctx, l, cmdCons)
 	if err != nil {
 		// Don't wrap the error since it's informative enough as is.
 		return err
 	}
 
+	basePath := filepath.Join(c.localesDir, defaultBaseFile)
 	if slices.Contains(dels, basePath) {
 		return errors.Error("base locale contains deletions")
 	}
@@ -370,15 +368,15 @@ func handleDels(
 
 // changedLocales returns cleaned paths of locales with changes or error.  adds
 // is the list of locales with only additions.  dels is the list of locales
-// with only deletions.
-func changedLocales(
+// with only deletions.  l and cmdCons must not be nil.
+func (c *twoskyClient) changedLocales(
 	ctx context.Context,
 	l *slog.Logger,
 	cmdCons executil.CommandConstructor,
 ) (adds, dels []string, err error) {
 	defer func() { err = errors.Annotate(err, "getting changes: %w") }()
 
-	gitArgs := []string{"diff", "--numstat", localesDirHome}
+	gitArgs := []string{"diff", "--numstat", c.localesDir}
 	l.DebugContext(ctx, "executing", "cmd", gitCmd, "args", gitArgs)
 
 	// TODO(s.chzhen):  Consider streaming the output if needed.  Using

scripts/translations/twosky.go

@@ -13,30 +13,20 @@ import (
 	"github.com/AdguardTeam/golibs/validate"
 )
 
-// Constants for mapping the twosky configurations.
-//
-// Keep in sync with the .twosky.json file.
-const (
-	// twoskyProjectIdxHome is the index of the Home project in the localization
-	// configuration.
-	twoskyProjectIdxHome = iota
-
-	// twoskyProjectIdxServices is the index of the Services project in the
-	// localization configuration.
-	twoskyProjectIdxServices
-
-	// twoskyProjectCount is the number of projects in the localization
-	// configuration.
-	twoskyProjectCount
-)
+// metadata represents additional information for a single project.
+type metadata struct {
+	LocalesDir string `json:"locales_dir"`
+	SourcesDir string `json:"sources_dir"`
+}
 
 // twoskyConfig is the configuration structure for localization of a single
 // project.
 type twoskyConfig struct {
 	Languages        languages `json:"languages"`
 	ProjectID        string    `json:"project_id"`
-	BaseLangcode     langCode  `json:"base_locale"`
+	BaseLangCode     langCode  `json:"base_locale"`
 	LocalizableFiles []string  `json:"localizable_files"`
+	Metadata         metadata  `json:"metadata"`
 }
 
 // type check
@@ -50,8 +40,11 @@ func (t *twoskyConfig) Validate() (err error) {
 
 	errs := []error{
 		validate.NotEmpty("project_id", t.ProjectID),
-		validate.NotEmpty("base_locale", t.BaseLangcode),
+		validate.NotEmpty("base_locale", t.BaseLangCode),
 		validate.NotEmptySlice("localizable_files", t.LocalizableFiles),
+		validate.NotEmpty("metadata", t.Metadata),
+		validate.NotEmpty("metadata.locales_dir", t.Metadata.LocalesDir),
+		validate.NotEmpty("metadata.sources_dir", t.Metadata.SourcesDir),
 	}
 
 	if len(t.Languages) == 0 {
@@ -68,48 +61,63 @@ func (t *twoskyConfig) Validate() (err error) {
 	return errors.Join(errs...)
 }
 
-// readTwoskyConfig returns twosky configuration.
-func readTwoskyConfig() (home, services *twoskyConfig, err error) {
+// newTwoskyConfig returns new twosky configuration.
+func newTwoskyConfig() (conf *twoskyConfig, err error) {
 	defer func() { err = errors.Annotate(err, "parsing twosky config: %w") }()
 
 	b, err := os.ReadFile(twoskyConfFile)
 	if err != nil {
 		// Don't wrap the error since it's informative enough as is.
-		return nil, nil, err
+		return nil, err
 	}
 
-	var tsc []*twoskyConfig
-	err = json.Unmarshal(b, &tsc)
+	var confs []*twoskyConfig
+	err = json.Unmarshal(b, &confs)
 	if err != nil {
-		return nil, nil, fmt.Errorf("unmarshalling %q: %w", twoskyConfFile, err)
+		return nil, fmt.Errorf("unmarshalling %q: %w", twoskyConfFile, err)
 	}
 
-	err = validate.Equal("projects count", len(tsc), twoskyProjectCount)
+	err = validate.NotEmptySlice("projects", confs)
 	if err != nil {
-		return nil, nil, err
+		// Don't wrap the error since it's informative enough as is.
+		return nil, err
 	}
 
-	err = errors.Join(validate.AppendSlice(nil, "projects", tsc)...)
+	err = errors.Join(validate.AppendSlice(nil, "projects", confs)...)
 	if err != nil {
-		return nil, nil, err
+		// Don't wrap the error since it's informative enough as is.
+		return nil, err
 	}
 
-	return tsc[twoskyProjectIdxHome], tsc[twoskyProjectIdxServices], nil
+	projectID := cmp.Or(os.Getenv("TWOSKY_PROJECT_ID"), defaultProjectID)
+	for _, c := range confs {
+		if c.ProjectID == projectID {
+			return c, nil
+		}
+	}
+
+	return nil, fmt.Errorf("project %q not found in %s", projectID, twoskyConfFile)
 }
 
-// twoskyClient is the twosky client with methods for download and upload
-// translations.
+// twoskyClient is the client for the twosky translation service.
 type twoskyClient struct {
 	// uri is the base URL.
 	uri *url.URL
 
-	// projectID is the name of the project.
-	projectID string
-
 	// baseLang is the base language code.
 	baseLang langCode
 
-	// langs is the list of codes of languages to download.
+	// localesDir is the path to the directory with locale files.
+	localesDir string
+
+	// projectID is the name of the project.
+	projectID string
+
+	// sourcesDir is the path to directory with source files in there the
+	// localizations are used.
+	sourcesDir string
+
+	// langs is the list of codes of languages.
 	langs []langCode
 
 	// localizableFiles are the files to localize.
@@ -117,7 +125,7 @@ type twoskyClient struct {
 }
 
 // newTwoskyClient reads values from environment variables or defaults,
-// validates them, and returns the twosky client.
+// validates them, and returns the twosky client.  conf must be valid.
 func newTwoskyClient(conf *twoskyConfig) (cli *twoskyClient, err error) {
 	defer func() { err = errors.Annotate(err, "filling config: %w") }()
 
@@ -134,7 +142,7 @@ func newTwoskyClient(conf *twoskyConfig) (cli *twoskyClient, err error) {
 		projectID = envProjectID
 	}
 
-	baseLang := conf.BaseLangcode
+	baseLang := conf.BaseLangCode
 	uLangStr := os.Getenv("UPLOAD_LANGUAGE")
 	if uLangStr != "" {
 		baseLang = langCode(uLangStr)
@@ -157,8 +165,10 @@ func newTwoskyClient(conf *twoskyConfig) (cli *twoskyClient, err error) {
 
 	return &twoskyClient{
 		uri:              uri,
-		projectID:        projectID,
 		baseLang:         baseLang,
+		localesDir:       conf.Metadata.LocalesDir,
+		projectID:        projectID,
+		sourcesDir:       conf.Metadata.SourcesDir,
 		langs:            langs,
 		localizableFiles: conf.LocalizableFiles,
 	}, nil

scripts/translations/upload.go

@@ -22,7 +22,7 @@ func (c *twoskyClient) upload() (err error) {
 	defer func() { err = errors.Annotate(err, "upload: %w") }()
 
 	uploadURI := c.uri.JoinPath("upload")
-	basePath := filepath.Join(localesDirHome, defaultBaseFile)
+	basePath := filepath.Join(c.localesDir, defaultBaseFile)
 
 	formData := map[string]string{
 		"format":   "json",