4b3cf3dc创建于 2025年8月8日历史提交
/*
 * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved.
 */
package magic.examples.doc_translator

import magic.config.Config
import magic.utils.{readFile, writeFile, Color, Colorful}

import std.env.*
import std.argopt.*
import std.collection.ArrayList
import std.fs.{Path, FileInfo, Directory, canonicalize, exists}

var SOURCE_PATH = Path(".")
var TARGET_PATH = Path(".")

private func translate(content: String): String {
    let translator = Translator()
    // A file may exceed the LLM context, so we should split it as chunks,
    // and translate these chunks
    let lines = content.split("\n")
    let CHUNK_SIZE = 300 // If the chunk size is too long, LLM is error-prone
    var count = 1
    let result = ArrayList<String>()
    for (start in 0..lines.size:CHUNK_SIZE) {
        println("  Chunks: ${count}/${(lines.size + CHUNK_SIZE - 1) / CHUNK_SIZE}")
        var end = start + CHUNK_SIZE
        if (end > lines.size) {
            end = lines.size
        }
        let chunk = String.join(lines[start..end], delimiter: "\n")
        result.add(translator.chat(chunk))
        count += 1
    }
    return String.join(result.toArray(), delimiter: "\n")
}

private func translateDoc(docPath: Path, newPath: Path): Unit {
    let doc = readFile(docPath)
    println("Process the doc: ${docPath.toString().withColor(Color.Blue)}")
    let newDoc = translate(doc)
    println(newDoc)
    println("Save to ==> ${newPath.toString().withColor(Color.Green)}")
    writeFile(newPath, newDoc)
}

private func buildNewPath(docPath: Path): Path {
    // Since all paths are absolute, we replace the prefix directly
    let newPath = Path(
        docPath.toString().replace(
            SOURCE_PATH.toString(),
            TARGET_PATH.toString()
        )
    )
    // If the parent dir does not exist, create it
    if (!exists(newPath.parent)) {
        Directory.create(newPath.parent, recursive: true)
    }
    return newPath
}

private func translateFilesUnderDir(dirPath: Path, ext!: String): Unit {
    Directory.walk(dirPath) { fileInfo =>
        let path = canonicalize(fileInfo.path)
        if (fileInfo.isRegular() && path.extensionName == ext) {
            println(path)
            let newPath = buildNewPath(path)
            translateDoc(path, newPath)
        } else if (fileInfo.isDirectory()) {
            translateFilesUnderDir(path, ext: ext)
        }
        return true
    }
}

//===================================================================

private func parseArgs(args: Array<String>): Unit {
    let argSpecs = [
        Long("source", RequiredValue),
        Long("target", RequiredValue)
    ]
    try {
        let parsedArgs = parseArguments(args, argSpecs)
        let source = parsedArgs.options.get("source").getOrThrow()
        let target = parsedArgs.options.get("target").getOrThrow()
        SOURCE_PATH = canonicalize(source)
        TARGET_PATH = Path(target)
    } catch (_: ArgumentParseException) {
        dumpManual()
        exit(1)
    } catch (_: NoneValueException) {
        dumpManual()
        exit(1)
    }
}

private func dumpManual(): Unit {
    println("Magic Translator:")
    println('cjpm run --run-args "--source <path-to-markdown> --target <path-to-new-file>"')
}

main(args: Array<String>): Unit {
    Config.logLevel = "DEBUG"
    Config.logFile = "./logs/abc.log"
    // Config.defaultContextLen = 32000

    parseArgs(args)
    let fileInfo = FileInfo(SOURCE_PATH)
    if (fileInfo.isRegular()) {
        translateDoc(SOURCE_PATH, TARGET_PATH)
    } else {
        // If the target dir does not exist, create it
        if (!exists(TARGET_PATH)) {
            Directory.create(TARGET_PATH, recursive: true)
        }
        translateFilesUnderDir(SOURCE_PATH, ext: "md")
    }
}