| #include "SourcePos.h" |
| #include "ValuesFile.h" |
| #include "XLIFFFile.h" |
| #include "Perforce.h" |
| #include "merge_res_and_xliff.h" |
| #include "localize.h" |
| #include "file_utils.h" |
| #include "res_check.h" |
| #include "xmb.h" |
| |
| #include <host/pseudolocalize.h> |
| |
| #include <stdlib.h> |
| #include <stdarg.h> |
| #include <sstream> |
| #include <stdio.h> |
| #include <string.h> |
| #include <stdlib.h> |
| |
| using namespace std; |
| |
| FILE* g_logFile = NULL; |
| |
| int test(); |
| |
| int |
| read_settings(const string& filename, map<string,Settings>* result, const string& rootDir) |
| { |
| XMLNode* root = NodeHandler::ParseFile(filename, XMLNode::PRETTY); |
| if (root == NULL) { |
| SourcePos(filename, -1).Error("Error reading file."); |
| return 1; |
| } |
| |
| // <configuration> |
| vector<XMLNode*> configNodes = root->GetElementsByName("", "configuration"); |
| const size_t I = configNodes.size(); |
| for (size_t i=0; i<I; i++) { |
| const XMLNode* configNode = configNodes[i]; |
| |
| Settings settings; |
| settings.id = configNode->GetAttribute("", "id", ""); |
| if (settings.id == "") { |
| configNode->Position().Error("<configuration> needs an id attribute."); |
| delete root; |
| return 1; |
| } |
| |
| settings.oldVersion = configNode->GetAttribute("", "old-cl", ""); |
| |
| settings.currentVersion = configNode->GetAttribute("", "new-cl", ""); |
| if (settings.currentVersion == "") { |
| configNode->Position().Error("<configuration> needs a new-cl attribute."); |
| delete root; |
| return 1; |
| } |
| |
| // <app> |
| vector<XMLNode*> appNodes = configNode->GetElementsByName("", "app"); |
| |
| const size_t J = appNodes.size(); |
| for (size_t j=0; j<J; j++) { |
| const XMLNode* appNode = appNodes[j]; |
| |
| string dir = appNode->GetAttribute("", "dir", ""); |
| if (dir == "") { |
| appNode->Position().Error("<app> needs a dir attribute."); |
| delete root; |
| return 1; |
| } |
| |
| settings.apps.push_back(dir); |
| } |
| |
| // <reject> |
| vector<XMLNode*> rejectNodes = configNode->GetElementsByName("", "reject"); |
| |
| const size_t K = rejectNodes.size(); |
| for (size_t k=0; k<K; k++) { |
| const XMLNode* rejectNode = rejectNodes[k]; |
| |
| Reject reject; |
| |
| reject.file = rejectNode->GetAttribute("", "file", ""); |
| if (reject.file == "") { |
| rejectNode->Position().Error("<reject> needs a file attribute."); |
| delete root; |
| return 1; |
| } |
| string f = reject.file; |
| reject.file = rootDir; |
| reject.file += '/'; |
| reject.file += f; |
| |
| reject.name = rejectNode->GetAttribute("", "name", ""); |
| if (reject.name == "") { |
| rejectNode->Position().Error("<reject> needs a name attribute."); |
| delete root; |
| return 1; |
| } |
| |
| reject.comment = trim_string(rejectNode->CollapseTextContents()); |
| |
| settings.reject.push_back(reject); |
| } |
| |
| (*result)[settings.id] = settings; |
| } |
| |
| delete root; |
| return 0; |
| } |
| |
| |
| static void |
| ValuesFile_to_XLIFFFile(const ValuesFile* values, XLIFFFile* xliff, const string& englishFilename) |
| { |
| const set<StringResource>& strings = values->GetStrings(); |
| for (set<StringResource>::const_iterator it=strings.begin(); it!=strings.end(); it++) { |
| StringResource res = *it; |
| res.file = englishFilename; |
| xliff->AddStringResource(res); |
| } |
| } |
| |
| static bool |
| contains_reject(const Settings& settings, const string& file, const TransUnit& tu) |
| { |
| const string name = tu.id; |
| const vector<Reject>& reject = settings.reject; |
| const size_t I = reject.size(); |
| for (size_t i=0; i<I; i++) { |
| const Reject& r = reject[i]; |
| if (r.file == file && r.name == name) { |
| return true; |
| } |
| } |
| return false; |
| } |
| |
| /** |
| * If it's been rejected, then we keep whatever info we have. |
| * |
| * Implements this truth table: |
| * |
| * S AT AS Keep |
| * ----------------------- |
| * 0 0 0 0 (this case can't happen) |
| * 0 0 1 0 (it was there, never translated, and removed) |
| * 0 1 0 0 (somehow it got translated, but it was removed) |
| * 0 1 1 0 (it was removed after having been translated) |
| * |
| * 1 0 0 1 (it was just added) |
| * 1 0 1 1 (it was added, has been changed, but it never got translated) |
| * 1 1 0 1 (somehow it got translated, but we don't know based on what) |
| * 1 1 1 0/1 (it's in both. 0 if S=AS b/c there's no need to retranslate if they're |
| * the same. 1 if S!=AS because S changed, so it should be retranslated) |
| * |
| * The first four are cases where, whatever happened in the past, the string isn't there |
| * now, so it shouldn't be in the XLIFF file. |
| * |
| * For cases 4 and 5, the string has never been translated, so get it translated. |
| * |
| * For case 6, it's unclear where the translated version came from, so we're conservative |
| * and send it back for them to have another shot at. |
| * |
| * For case 7, we have some data. We have two choices. We could rely on the translator's |
| * translation memory or tools to notice that the strings haven't changed, and populate the |
| * <target> field themselves. Or if the string hasn't changed since last time, we can just |
| * not even tell them about it. As the project nears the end, it will be convenient to see |
| * the xliff files reducing in size, so we pick the latter. Obviously, if the string has |
| * changed, then we need to get it retranslated. |
| */ |
| bool |
| keep_this_trans_unit(const string& file, const TransUnit& unit, void* cookie) |
| { |
| const Settings* settings = reinterpret_cast<const Settings*>(cookie); |
| |
| if (contains_reject(*settings, file, unit)) { |
| return true; |
| } |
| |
| if (unit.source.id == "") { |
| return false; |
| } |
| if (unit.altTarget.id == "" || unit.altSource.id == "") { |
| return true; |
| } |
| return unit.source.value->ContentsToString(XLIFF_NAMESPACES) |
| != unit.altSource.value->ContentsToString(XLIFF_NAMESPACES); |
| } |
| |
| int |
| validate_config(const string& settingsFile, const map<string,Settings>& settings, |
| const string& config) |
| { |
| if (settings.find(config) == settings.end()) { |
| SourcePos(settingsFile, -1).Error("settings file does not contain setting: %s\n", |
| config.c_str()); |
| return 1; |
| } |
| return 0; |
| } |
| |
| int |
| validate_configs(const string& settingsFile, const map<string,Settings>& settings, |
| const vector<string>& configs) |
| { |
| int err = 0; |
| for (size_t i=0; i<configs.size(); i++) { |
| string config = configs[i]; |
| err |= validate_config(settingsFile, settings, config); |
| } |
| return err; |
| } |
| |
| int |
| select_files(vector<string> *resFiles, const string& config, |
| const map<string,Settings>& settings, const string& rootDir) |
| { |
| int err; |
| vector<vector<string> > allResFiles; |
| vector<string> configs; |
| configs.push_back(config); |
| err = select_files(&allResFiles, configs, settings, rootDir); |
| if (err == 0) { |
| *resFiles = allResFiles[0]; |
| } |
| return err; |
| } |
| |
| int |
| select_files(vector<vector<string> > *allResFiles, const vector<string>& configs, |
| const map<string,Settings>& settings, const string& rootDir) |
| { |
| int err; |
| printf("Selecting files..."); |
| fflush(stdout); |
| |
| for (size_t i=0; i<configs.size(); i++) { |
| const string& config = configs[i]; |
| const Settings& setting = settings.find(config)->second; |
| |
| vector<string> resFiles; |
| err = Perforce::GetResourceFileNames(setting.currentVersion, rootDir, |
| setting.apps, &resFiles, true); |
| if (err != 0) { |
| fprintf(stderr, "error with perforce. bailing\n"); |
| return err; |
| } |
| |
| allResFiles->push_back(resFiles); |
| } |
| return 0; |
| } |
| |
| static int |
| do_export(const string& settingsFile, const string& rootDir, const string& outDir, |
| const string& targetLocale, const vector<string>& configs) |
| { |
| bool success = true; |
| int err; |
| |
| if (false) { |
| printf("settingsFile=%s\n", settingsFile.c_str()); |
| printf("rootDir=%s\n", rootDir.c_str()); |
| printf("outDir=%s\n", outDir.c_str()); |
| for (size_t i=0; i<configs.size(); i++) { |
| printf("config[%zd]=%s\n", i, configs[i].c_str()); |
| } |
| } |
| |
| map<string,Settings> settings; |
| err = read_settings(settingsFile, &settings, rootDir); |
| if (err != 0) { |
| return err; |
| } |
| |
| err = validate_configs(settingsFile, settings, configs); |
| if (err != 0) { |
| return err; |
| } |
| |
| vector<vector<string> > allResFiles; |
| err = select_files(&allResFiles, configs, settings, rootDir); |
| if (err != 0) { |
| return err; |
| } |
| |
| size_t totalFileCount = 0; |
| for (size_t i=0; i<allResFiles.size(); i++) { |
| totalFileCount += allResFiles[i].size(); |
| } |
| totalFileCount *= 3; // we try all 3 versions of the file |
| |
| size_t fileProgress = 0; |
| vector<Stats> stats; |
| vector<pair<string,XLIFFFile*> > xliffs; |
| |
| for (size_t i=0; i<configs.size(); i++) { |
| const string& config = configs[i]; |
| const Settings& setting = settings[config]; |
| |
| if (false) { |
| fprintf(stderr, "Configuration: %s (%zd of %zd)\n", config.c_str(), i+1, |
| configs.size()); |
| fprintf(stderr, " Old CL: %s\n", setting.oldVersion.c_str()); |
| fprintf(stderr, " Current CL: %s\n", setting.currentVersion.c_str()); |
| } |
| |
| Configuration english; |
| english.locale = "en_US"; |
| Configuration translated; |
| translated.locale = targetLocale; |
| XLIFFFile* xliff = XLIFFFile::Create(english, translated, setting.currentVersion); |
| |
| const vector<string>& resFiles = allResFiles[i]; |
| const size_t J = resFiles.size(); |
| for (size_t j=0; j<J; j++) { |
| string resFile = resFiles[j]; |
| |
| // parse the files into a ValuesFile |
| // pull out the strings and add them to the XLIFFFile |
| |
| // current file |
| print_file_status(++fileProgress, totalFileCount); |
| ValuesFile* currentFile = get_values_file(resFile, english, CURRENT_VERSION, |
| setting.currentVersion, true); |
| if (currentFile != NULL) { |
| ValuesFile_to_XLIFFFile(currentFile, xliff, resFile); |
| //printf("currentFile=[%s]\n", currentFile->ToString().c_str()); |
| } else { |
| fprintf(stderr, "error reading file %s@%s\n", resFile.c_str(), |
| setting.currentVersion.c_str()); |
| success = false; |
| } |
| |
| // old file |
| print_file_status(++fileProgress, totalFileCount); |
| ValuesFile* oldFile = get_values_file(resFile, english, OLD_VERSION, |
| setting.oldVersion, false); |
| if (oldFile != NULL) { |
| ValuesFile_to_XLIFFFile(oldFile, xliff, resFile); |
| //printf("oldFile=[%s]\n", oldFile->ToString().c_str()); |
| } |
| |
| // translated version |
| // (get the head of the tree for the most recent translation, but it's considered |
| // the old one because the "current" one hasn't been made yet, and this goes into |
| // the <alt-trans> tag if necessary |
| print_file_status(++fileProgress, totalFileCount); |
| string transFilename = translated_file_name(resFile, targetLocale); |
| ValuesFile* transFile = get_values_file(transFilename, translated, OLD_VERSION, |
| setting.currentVersion, false); |
| if (transFile != NULL) { |
| ValuesFile_to_XLIFFFile(transFile, xliff, resFile); |
| } |
| |
| delete currentFile; |
| delete oldFile; |
| delete transFile; |
| } |
| |
| Stats beforeFilterStats = xliff->GetStats(config); |
| |
| // run through the XLIFFFile and strip out TransUnits that have identical |
| // old and current source values and are not in the reject list, or just |
| // old values and no source values |
| xliff->Filter(keep_this_trans_unit, (void*)&setting); |
| |
| Stats afterFilterStats = xliff->GetStats(config); |
| afterFilterStats.totalStrings = beforeFilterStats.totalStrings; |
| |
| // add the reject comments |
| for (vector<Reject>::const_iterator reject = setting.reject.begin(); |
| reject != setting.reject.end(); reject++) { |
| TransUnit* tu = xliff->EditTransUnit(reject->file, reject->name); |
| tu->rejectComment = reject->comment; |
| } |
| |
| // config-locale-current_cl.xliff |
| stringstream filename; |
| if (outDir != "") { |
| filename << outDir << '/'; |
| } |
| filename << config << '-' << targetLocale << '-' << setting.currentVersion << ".xliff"; |
| xliffs.push_back(pair<string,XLIFFFile*>(filename.str(), xliff)); |
| |
| stats.push_back(afterFilterStats); |
| } |
| |
| // today is a good day to die |
| if (!success || SourcePos::HasErrors()) { |
| return 1; |
| } |
| |
| // write the XLIFF files |
| printf("\nWriting %zd file%s...\n", xliffs.size(), xliffs.size() == 1 ? "" : "s"); |
| for (vector<pair<string,XLIFFFile*> >::iterator it = xliffs.begin(); it != xliffs.end(); it++) { |
| const string& filename = it->first; |
| XLIFFFile* xliff = it->second; |
| string text = xliff->ToString(); |
| write_to_file(filename, text); |
| } |
| |
| // the stats |
| printf("\n" |
| " to without total\n" |
| " config files translate comments strings\n" |
| "-----------------------------------------------------------------------\n"); |
| Stats totals; |
| totals.config = "total"; |
| totals.files = 0; |
| totals.toBeTranslated = 0; |
| totals.noComments = 0; |
| totals.totalStrings = 0; |
| for (vector<Stats>::iterator it=stats.begin(); it!=stats.end(); it++) { |
| string cfg = it->config; |
| if (cfg.length() > 20) { |
| cfg.resize(20); |
| } |
| printf(" %-20s %-9zd %-9zd %-9zd %-19zd\n", cfg.c_str(), it->files, |
| it->toBeTranslated, it->noComments, it->totalStrings); |
| totals.files += it->files; |
| totals.toBeTranslated += it->toBeTranslated; |
| totals.noComments += it->noComments; |
| totals.totalStrings += it->totalStrings; |
| } |
| if (stats.size() > 1) { |
| printf("-----------------------------------------------------------------------\n" |
| " %-20s %-9zd %-9zd %-9zd %-19zd\n", totals.config.c_str(), totals.files, |
| totals.toBeTranslated, totals.noComments, totals.totalStrings); |
| } |
| printf("\n"); |
| return 0; |
| } |
| |
| struct PseudolocalizeSettings { |
| XLIFFFile* xliff; |
| bool expand; |
| }; |
| |
| |
| string |
| pseudolocalize_string(const string& source, const PseudolocalizeSettings* settings) |
| { |
| return pseudolocalize_string(source); |
| } |
| |
| static XMLNode* |
| pseudolocalize_xml_node(const XMLNode* source, const PseudolocalizeSettings* settings) |
| { |
| if (source->Type() == XMLNode::TEXT) { |
| return XMLNode::NewText(source->Position(), pseudolocalize_string(source->Text(), settings), |
| source->Pretty()); |
| } else { |
| XMLNode* target; |
| if (source->Namespace() == XLIFF_XMLNS && source->Name() == "g") { |
| // XXX don't translate these |
| target = XMLNode::NewElement(source->Position(), source->Namespace(), |
| source->Name(), source->Attributes(), source->Pretty()); |
| } else { |
| target = XMLNode::NewElement(source->Position(), source->Namespace(), |
| source->Name(), source->Attributes(), source->Pretty()); |
| } |
| |
| const vector<XMLNode*>& children = source->Children(); |
| const size_t I = children.size(); |
| for (size_t i=0; i<I; i++) { |
| target->EditChildren().push_back(pseudolocalize_xml_node(children[i], settings)); |
| } |
| |
| return target; |
| } |
| } |
| |
| void |
| pseudolocalize_trans_unit(const string&file, TransUnit* unit, void* cookie) |
| { |
| const PseudolocalizeSettings* settings = (PseudolocalizeSettings*)cookie; |
| |
| const StringResource& source = unit->source; |
| StringResource* target = &unit->target; |
| *target = source; |
| |
| target->config = settings->xliff->TargetConfig(); |
| |
| delete target->value; |
| target->value = pseudolocalize_xml_node(source.value, settings); |
| } |
| |
| int |
| pseudolocalize_xliff(XLIFFFile* xliff, bool expand) |
| { |
| PseudolocalizeSettings settings; |
| |
| settings.xliff = xliff; |
| settings.expand = expand; |
| xliff->Map(pseudolocalize_trans_unit, &settings); |
| return 0; |
| } |
| |
| static int |
| do_pseudo(const string& infile, const string& outfile, bool expand) |
| { |
| int err; |
| |
| XLIFFFile* xliff = XLIFFFile::Parse(infile); |
| if (xliff == NULL) { |
| return 1; |
| } |
| |
| pseudolocalize_xliff(xliff, expand); |
| |
| err = write_to_file(outfile, xliff->ToString()); |
| |
| delete xliff; |
| |
| return err; |
| } |
| |
| void |
| log_printf(const char *fmt, ...) |
| { |
| int ret; |
| va_list ap; |
| |
| if (g_logFile != NULL) { |
| va_start(ap, fmt); |
| ret = vfprintf(g_logFile, fmt, ap); |
| va_end(ap); |
| fflush(g_logFile); |
| } |
| } |
| |
| void |
| close_log_file() |
| { |
| if (g_logFile != NULL) { |
| fclose(g_logFile); |
| } |
| } |
| |
| void |
| open_log_file(const char* file) |
| { |
| g_logFile = fopen(file, "w"); |
| printf("log file: %s -- %p\n", file, g_logFile); |
| atexit(close_log_file); |
| } |
| |
| static int |
| usage() |
| { |
| fprintf(stderr, |
| "usage: localize export OPTIONS CONFIGS...\n" |
| " REQUIRED OPTIONS\n" |
| " --settings SETTINGS The settings file to use. See CONFIGS below.\n" |
| " --root TREE_ROOT The location in Perforce of the files. e.g. //device\n" |
| " --target LOCALE The target locale. See LOCALES below.\n" |
| "\n" |
| " OPTIONAL OPTIONS\n" |
| " --out DIR Directory to put the output files. Defaults to the\n" |
| " current directory if not supplied. Files are\n" |
| " named as follows:\n" |
| " CONFIG-LOCALE-CURRENT_CL.xliff\n" |
| "\n" |
| "\n" |
| "usage: localize import XLIFF_FILE...\n" |
| "\n" |
| "Import a translated XLIFF file back into the tree.\n" |
| "\n" |
| "\n" |
| "usage: localize xlb XMB_FILE VALUES_FILES...\n" |
| "\n" |
| "Read resource files from the tree file and write the corresponding XLB file\n" |
| "\n" |
| "Supply all of the android resource files (values files) to export after that.\n" |
| "\n" |
| "\n" |
| "\n" |
| "CONFIGS\n" |
| "\n" |
| "LOCALES\n" |
| "Locales are specified in the form en_US They will be processed correctly\n" |
| "to locate the resouce files in the tree.\n" |
| "\n" |
| "\n" |
| "usage: localize pseudo OPTIONS INFILE [OUTFILE]\n" |
| " OPTIONAL OPTIONS\n" |
| " --big Pad strings so they get longer.\n" |
| "\n" |
| "Read INFILE, an XLIFF file, and output a pseudotranslated version of that file. If\n" |
| "OUTFILE is specified, the results are written there; otherwise, the results are\n" |
| "written back to INFILE.\n" |
| "\n" |
| "\n" |
| "usage: localize rescheck FILES...\n" |
| "\n" |
| "Reads the base strings and prints warnings about bad resources from the given files.\n" |
| "\n"); |
| return 1; |
| } |
| |
| int |
| main(int argc, const char** argv) |
| { |
| //open_log_file("log.txt"); |
| //g_logFile = stdout; |
| |
| if (argc == 2 && 0 == strcmp(argv[1], "--test")) { |
| return test(); |
| } |
| |
| if (argc < 2) { |
| return usage(); |
| } |
| |
| int index = 1; |
| |
| if (0 == strcmp("export", argv[index])) { |
| string settingsFile; |
| string rootDir; |
| string outDir; |
| string baseLocale = "en"; |
| string targetLocale; |
| string language, region; |
| vector<string> configs; |
| |
| index++; |
| while (index < argc) { |
| if (0 == strcmp("--settings", argv[index])) { |
| settingsFile = argv[index+1]; |
| index += 2; |
| } |
| else if (0 == strcmp("--root", argv[index])) { |
| rootDir = argv[index+1]; |
| index += 2; |
| } |
| else if (0 == strcmp("--out", argv[index])) { |
| outDir = argv[index+1]; |
| index += 2; |
| } |
| else if (0 == strcmp("--target", argv[index])) { |
| targetLocale = argv[index+1]; |
| index += 2; |
| } |
| else if (argv[index][0] == '-') { |
| fprintf(stderr, "unknown argument %s\n", argv[index]); |
| return usage(); |
| } |
| else { |
| break; |
| } |
| } |
| for (; index<argc; index++) { |
| configs.push_back(argv[index]); |
| } |
| |
| if (settingsFile == "" || rootDir == "" || configs.size() == 0 || targetLocale == "") { |
| return usage(); |
| } |
| if (!split_locale(targetLocale, &language, ®ion)) { |
| fprintf(stderr, "illegal --target locale: '%s'\n", targetLocale.c_str()); |
| return usage(); |
| } |
| |
| |
| return do_export(settingsFile, rootDir, outDir, targetLocale, configs); |
| } |
| else if (0 == strcmp("import", argv[index])) { |
| vector<string> xliffFilenames; |
| |
| index++; |
| for (; index<argc; index++) { |
| xliffFilenames.push_back(argv[index]); |
| } |
| |
| return do_merge(xliffFilenames); |
| } |
| else if (0 == strcmp("xlb", argv[index])) { |
| string outfile; |
| vector<string> resFiles; |
| |
| index++; |
| if (argc < index+1) { |
| return usage(); |
| } |
| |
| outfile = argv[index]; |
| |
| index++; |
| for (; index<argc; index++) { |
| resFiles.push_back(argv[index]); |
| } |
| |
| return do_xlb_export(outfile, resFiles); |
| } |
| else if (0 == strcmp("pseudo", argv[index])) { |
| string infile; |
| string outfile; |
| bool big = false; |
| |
| index++; |
| while (index < argc) { |
| if (0 == strcmp("--big", argv[index])) { |
| big = true; |
| index += 1; |
| } |
| else if (argv[index][0] == '-') { |
| fprintf(stderr, "unknown argument %s\n", argv[index]); |
| return usage(); |
| } |
| else { |
| break; |
| } |
| } |
| |
| if (index == argc-1) { |
| infile = argv[index]; |
| outfile = argv[index]; |
| } |
| else if (index == argc-2) { |
| infile = argv[index]; |
| outfile = argv[index+1]; |
| } |
| else { |
| fprintf(stderr, "unknown argument %s\n", argv[index]); |
| return usage(); |
| } |
| |
| return do_pseudo(infile, outfile, big); |
| } |
| else if (0 == strcmp("rescheck", argv[index])) { |
| vector<string> files; |
| |
| index++; |
| while (index < argc) { |
| if (argv[index][0] == '-') { |
| fprintf(stderr, "unknown argument %s\n", argv[index]); |
| return usage(); |
| } |
| else { |
| break; |
| } |
| } |
| for (; index<argc; index++) { |
| files.push_back(argv[index]); |
| } |
| |
| if (files.size() == 0) { |
| return usage(); |
| } |
| |
| return do_rescheck(files); |
| } |
| else { |
| return usage(); |
| } |
| |
| if (SourcePos::HasErrors()) { |
| SourcePos::PrintErrors(stderr); |
| return 1; |
| } |
| |
| return 0; |
| } |
| |
