"""Generates .h and .rc files for strings extracted from a .grd file.
This script generates an rc file and header (NAME.{rc,h}) to be included in
a build target. The rc file includes translations for strings pulled from the
given .grd file(s) and their corresponding localized .xtb files.
To specify strings that will be extracted, the script pointed to by the
argument "extract-datafile" should contain one or both of the following global
variables:
STRING_IDS is a list of strings IDs we want to import from the .grd files and
include in the generated RC file. These strings are universal for all brands.
MODE_SPECIFIC_STRINGS: is a dictionary of strings for which there are brand
specific values. This mapping provides brand- and mode-specific string ids for a
given input id as described here:
{
resource_id_1: { # A resource ID for use with GetLocalizedString.
brand_1: [ # 'google_chrome', for example.
string_id_1, # Strings listed in order of the brand's modes, as
string_id_2, # specified in install_static::InstallConstantIndex.
...
string_id_N,
],
brand_2: [ # 'chromium', for example.
...
],
},
resource_id_2: ...
}
Note: MODE_SPECIFIC_STRINGS cannot be specified if STRING_IDS is not specified.
"""
from __future__ import print_function
import argparse
import collections
import glob
import io
import os
import sys
from xml import sax
BASEDIR = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(1, os.path.join(BASEDIR, '../../../tools/grit'))
sys.path.insert(2, os.path.join(BASEDIR, '../../../tools/python'))
from grit.extern import tclib
class GrdHandler(sax.handler.ContentHandler):
"""Extracts selected strings from a .grd file.
Attributes:
messages: A dict mapping string identifiers to their corresponding messages
(key "text") and transconsole ids (key "tc_id").
referenced_xtb_files: A list of all xtb files referenced inside the .grd
file.
"""
def __init__(self, string_id_set):
"""Constructs a handler that reads selected strings from a .grd file.
The dict attribute |messages| is populated with the strings that are read.
Args:
string_id_set: An optional set of message identifiers to extract; all
messages are extracted if empty.
"""
sax.handler.ContentHandler.__init__(self)
self.messages = collections.defaultdict(dict)
self.referenced_xtb_files = []
self.__id_set = string_id_set
self.__message_name = None
self.__element_stack = []
self.__text_scraps = []
self.__tc_text_scraps = []
self.__characters_callback = None
def startElement(self, name, attrs):
self.__element_stack.append(name)
if name == 'message':
self.__OnOpenMessage(attrs.getValue('name'))
elif name == 'ph':
self.__OnOpenPlaceholder(attrs.getValue('name'))
elif name == 'file':
parent = self.__element_stack[-2]
if parent == 'translations':
self.__OnAddXtbFile(attrs.getValue('path'))
def endElement(self, name):
popped = self.__element_stack.pop()
assert popped == name
if name == 'message':
self.__OnCloseMessage()
def characters(self, content):
if self.__characters_callback:
self.__characters_callback(self.__element_stack[-1], content)
def __IsExtractingMessage(self):
"""Returns True if a message is currently being extracted."""
return self.__message_name is not None
def __OnOpenMessage(self, message_name):
"""Invoked at the start of a <message> with message's name."""
assert not self.__IsExtractingMessage()
self.__message_name = (message_name if (not (self.__id_set) or
message_name in self.__id_set)
else None)
if self.__message_name:
self.__characters_callback = self.__OnMessageText
def __OnOpenPlaceholder(self, ph_name):
"""Invoked at the start of a <ph> with the `name` attribute."""
if self.__IsExtractingMessage():
self.__tc_text_scraps.append(ph_name)
def __OnMessageText(self, containing_element, message_text):
"""Invoked to handle a block of text for a message."""
if message_text and (containing_element == 'message' or
containing_element == 'ph'):
self.__text_scraps.append(message_text)
if containing_element == 'message':
self.__tc_text_scraps.append(message_text)
def __OnCloseMessage(self):
"""Invoked at the end of a message."""
if self.__IsExtractingMessage():
self.messages[self.__message_name]["text"] = ''.join(
self.__text_scraps).strip()
self.messages[self.__message_name]["tc_id"] = tclib.GenerateMessageId(
''.join(self.__tc_text_scraps).strip())
self.__message_name = None
self.__text_scraps = []
self.__tc_text_scraps = []
self.__characters_callback = None
def __OnAddXtbFile(self, xtb_file_path):
"""Adds the xtb file path of a 'file'."""
if os.path.splitext(xtb_file_path)[1].lower() == '.xtb':
self.referenced_xtb_files.append(xtb_file_path)
class XtbHandler(sax.handler.ContentHandler):
"""Extracts selected translations from an .xtd file.
Populates the |lang| and |translations| attributes with the language and
selected strings of an .xtb file. Instances may be re-used to read the same
set of translations from multiple .xtb files.
Attributes:
translations: A mapping of translation ids to strings.
lang: The language parsed from the .xtb file.
"""
def __init__(self, translation_ids):
"""Constructs an instance to parse the given strings from an .xtb file.
Args:
translation_ids: a mapping of translation ids to their string
identifiers list for the translations to be extracted.
"""
sax.handler.ContentHandler.__init__(self)
self.lang = None
self.translations = None
self.__translation_ids = translation_ids
self.__element_stack = []
self.__string_ids = None
self.__text_scraps = []
self.__ph_count = 0
self.__characters_callback = None
def startDocument(self):
self.lang = ''
self.translations = {}
def startElement(self, name, attrs):
self.__element_stack.append(name)
if len(self.__element_stack) == 1:
assert name == 'translationbundle'
self.__OnLanguage(attrs.getValue('lang'))
if name == 'translation':
self.__OnOpenTranslation(attrs.getValue('id'))
elif name == 'ph':
self.__OnOpenPlaceholder()
def endElement(self, name):
popped = self.__element_stack.pop()
assert popped == name
if name == 'translation':
self.__OnCloseTranslation()
def characters(self, content):
if self.__characters_callback:
self.__characters_callback(self.__element_stack[-1], content)
def __IsExtractingTranslation(self):
"""Returns `True` if a translation is currently being extracted."""
return self.__string_ids is not None
def __OnLanguage(self, lang):
self.lang = lang.replace('-', '_').upper()
def __OnOpenTranslation(self, translation_id):
assert not self.__IsExtractingTranslation()
self.__string_ids = self.__translation_ids.get(translation_id)
if self.__string_ids:
self.__characters_callback = self.__OnTranslationText
def __OnOpenPlaceholder(self):
if self.__IsExtractingTranslation():
self.__ph_count += 1
self.__text_scraps.append('$' + str(self.__ph_count))
def __OnTranslationText(self, containing_element, message_text):
if message_text and containing_element == 'translation':
self.__text_scraps.append(message_text)
def __OnCloseTranslation(self):
if self.__IsExtractingTranslation():
translated_string = ''.join(self.__text_scraps).strip()
for string_id in self.__string_ids:
self.translations[string_id] = translated_string
self.__string_ids = None
self.__text_scraps = []
self.__ph_count = 0
self.__characters_callback = None
class StringRcMaker(object):
"""Makes .h and .rc files containing strings and translations."""
def __init__(self, inputs, expected_xtb_input_files, header_file, rc_file,
brand, first_resource_id, string_ids_to_extract, mode_specific_strings):
"""Constructs a maker.
Args:
inputs: A list of (grd_file, xtb_dir) pairs containing the source data.
expected_xtb_input_files: A list of xtb files that are expected to exist
in the inputs folders. If there is a discrepency between what exists
and what is expected the script will fail.
header_file: The location of the header file to write containing all the
defined string IDs.
rc_file: The location of the rc file to write containing all the string
resources.
brand: The brand to check against when extracting mode-specific strings.
first_resource_id: The starting ID for the generated string resources.
string_ids_to_extract: The IDs of strings we want to import from the .grd
files and include in the generated RC file. These strings are universal
for all brands.
mode_specific_strings: A dictionary of strings that have conditional
values based on the brand's install mode. Refer to the documentation at
the top of this file for more information on the format of the
dictionary.
"""
self.inputs = inputs
self.expected_xtb_input_files = expected_xtb_input_files
self.expected_xtb_input_files.sort()
self.header_file = header_file
self.rc_file = rc_file
self.brand = brand
self.first_resource_id = first_resource_id;
self.string_id_set = set(string_ids_to_extract)
self.mode_specific_strings = mode_specific_strings
self.__AddModeSpecificStringIds()
def MakeFiles(self):
translated_strings = self.__ReadSourceAndTranslatedStrings()
self.__WriteRCFile(translated_strings)
self.__WriteHeaderFile(translated_strings)
class __TranslationData(object):
"""A container of information about a single translation."""
def __init__(self, resource_id_str, language, translation):
self.resource_id_str = resource_id_str
self.language = language
self.translation = translation
def __lt__(self, other):
"""Allow __TranslationDatas to be sorted by id then by language."""
return (self.resource_id_str, self.language) < (other.resource_id_str,
other.language)
def __AddModeSpecificStringIds(self):
"""Adds the mode-specific strings for all of the current brand's install
modes to self.string_id_set."""
for string_id, brands in self.mode_specific_strings.items():
brand_strings = brands.get(self.brand)
if not brand_strings:
raise RuntimeError(
'No strings declared for brand \'%s\' in MODE_SPECIFIC_STRINGS for '
'message %s' % (self.brand, string_id))
self.string_id_set.update(brand_strings)
def __ReadSourceAndTranslatedStrings(self):
"""Reads the source strings and translations from all inputs."""
translated_strings = []
all_xtb_files = []
for grd_file, xtb_dir in self.inputs:
source_name = os.path.splitext(os.path.basename(grd_file))[0]
xtb_pattern = os.path.join(os.path.dirname(grd_file), xtb_dir,
'%s*.xtb' % source_name)
local_xtb_files = [x.replace('\\', '/') for x in glob.glob(xtb_pattern)]
all_xtb_files.extend(local_xtb_files)
translated_strings.extend(
self.__ReadSourceAndTranslationsFrom(grd_file, local_xtb_files))
translated_strings.sort()
all_xtb_files.sort()
if self.expected_xtb_input_files != all_xtb_files:
extra = list(set(all_xtb_files) - set(self.expected_xtb_input_files))
missing = list(set(self.expected_xtb_input_files) - set(all_xtb_files))
error = '''Asserted file list does not match.
Expected input files:
{}
Actual input files:
{}
Missing input files:
{}
Extra input files:
{}
'''
print(error.format('\n'.join(self.expected_xtb_input_files),
'\n'.join(all_xtb_files), '\n'.join(missing),
'\n'.join(extra)))
sys.exit(1)
return translated_strings
def __ReadSourceAndTranslationsFrom(self, grd_file, xtb_files):
"""Reads source strings and translations for a .grd file.
Reads the source strings and all available translations for the messages
identified by self.string_id_set (or all the messages if self.string_id_set
is empty). The source string is used where translations are missing.
Args:
grd_file: Path to a .grd file.
xtb_files: List of paths to .xtb files.
Returns:
An unsorted list of __TranslationData instances.
"""
sax_parser = sax.make_parser()
grd_handler = GrdHandler(self.string_id_set)
sax_parser.setContentHandler(grd_handler)
sax_parser.parse(grd_file)
source_strings = grd_handler.messages
grd_file_path = os.path.dirname(grd_file)
source_xtb_files = []
for xtb_file in grd_handler.referenced_xtb_files:
relative_xtb_file_path = (
os.path.join(grd_file_path, xtb_file).replace('\\', '/'))
source_xtb_files.append(relative_xtb_file_path)
missing_xtb_files = list(set(source_xtb_files) - set(xtb_files))
translated_strings = []
for string_id, string_data in source_strings.items():
translated_strings.append(self.__TranslationData(string_id,
'EN_US',
string_data["text"]))
translation_ids = {}
for (string_id, string_data) in source_strings.items():
translation_ids.setdefault(string_data["tc_id"], []).append(string_id);
extra_xtb_files = []
xtb_handler = XtbHandler(translation_ids)
sax_parser.setContentHandler(xtb_handler)
for xtb_filename in xtb_files:
if not xtb_filename in source_xtb_files:
extra_xtb_files.append(xtb_filename)
sax_parser.parse(xtb_filename)
for string_id, string_data in source_strings.items():
translated_string = xtb_handler.translations.get(string_id,
string_data["text"])
translated_strings.append(self.__TranslationData(string_id,
xtb_handler.lang,
translated_string))
if missing_xtb_files or extra_xtb_files:
if missing_xtb_files:
missing_error = ("There were files that were found in the .grd file "
"'{}' but do not exist on disk:\n{}")
print(missing_error.format(grd_file, '\n'.join(missing_xtb_files)))
if extra_xtb_files:
extra_error = ("There were files that exist on disk but were not found "
"in the .grd file '{}':\n{}")
print(extra_error.format(grd_file, '\n'.join(extra_xtb_files)))
sys.exit(1)
return translated_strings
def __WriteRCFile(self, translated_strings):
"""Writes a resource file with the strings provided in |translated_strings|.
"""
HEADER_TEXT = (
u'#include "%s"\n\n'
u'STRINGTABLE\n'
u'BEGIN\n'
) % os.path.basename(self.header_file)
FOOTER_TEXT = (
u'END\n'
)
with io.open(self.rc_file,
mode='w',
encoding='utf-16',
newline='\n') as outfile:
outfile.write(HEADER_TEXT)
for translation in translated_strings:
escaped_text = (translation.translation.replace('"', '""')
.replace('\t', '\\t')
.replace('\n', '\\n'))
outfile.write(u' %s "%s"\n' %
(translation.resource_id_str + '_' + translation.language,
escaped_text))
outfile.write(FOOTER_TEXT)
def __WriteHeaderFile(self, translated_strings):
"""Writes a .h file with resource ids."""
lines = []
do_languages_lines = ['\n#define DO_LANGUAGES']
installer_string_mapping_lines = ['\n#define DO_STRING_MAPPING']
do_mode_strings_lines = ['\n#define DO_MODE_STRINGS']
seen_languages = set()
offset_id = 0
for translation_data in translated_strings:
lang = translation_data.language
if lang not in seen_languages:
seen_languages.add(lang)
lines.append('#define IDS_L10N_OFFSET_%s %s' % (lang, offset_id))
do_languages_lines.append(' HANDLE_LANGUAGE(%s, IDS_L10N_OFFSET_%s)'
% (lang.replace('_', '-').lower(), lang))
offset_id += 1
else:
break
resource_id = self.first_resource_id
for translation_data in translated_strings:
lines.append('#define %s %s' % (translation_data.resource_id_str + '_' +
translation_data.language,
resource_id))
resource_id += 1
for string_id, brands in self.mode_specific_strings.items():
brand_strings = brands.get(self.brand)
if not brand_strings:
raise RuntimeError(
'No strings declared for brand \'%s\' in MODE_SPECIFIC_STRINGS for '
'message %s' % (self.brand, string_id))
do_mode_strings_lines.append(
' HANDLE_MODE_STRING(%s_BASE, %s)'
% (string_id, ', '.join([ ('%s_BASE' % s) for s in brand_strings])))
if self.string_id_set:
string_ids_to_write = self.string_id_set;
else:
string_ids_to_write = {t.resource_id_str for t in translated_strings}
for string_id in sorted(string_ids_to_write):
lines.append('#define %s_BASE %s_%s' % (string_id,
string_id,
translated_strings[0].language))
installer_string_mapping_lines.append(' HANDLE_STRING(%s_BASE, %s)'
% (string_id, string_id))
with open(self.header_file, 'w') as outfile:
outfile.write('\n'.join(lines))
outfile.write('\n#ifndef RC_INVOKED')
outfile.write(' \\\n'.join(do_languages_lines))
outfile.write(' \\\n'.join(installer_string_mapping_lines))
outfile.write(' \\\n'.join(do_mode_strings_lines))
outfile.write('\n#endif // ndef RC_INVOKED\n')
def BuildArgumentParser():
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-b',
help='identifier of the browser brand (e.g., chromium).'
'This argument is mandatory if the module file included'
'by --extract-datafile contains MODE_SPECIFIC_STRINGS',
dest='brand')
parser.add_argument('-i', action='append',
required=True,
help='path to .grd file',
dest='input_grd_files')
parser.add_argument('-r', action='append',
required=True,
help='relative path to .xtb dir for each .grd file',
dest='input_xtb_relative_paths')
parser.add_argument('-x', action='append',
required=True,
help='expected xtb input files to read',
dest='expected_xtb_input_files')
parser.add_argument('--header-file',
required=True,
help='path to generated .h file to write',
dest='header_file')
parser.add_argument('--rc-file',
required=True,
help='path to generated .rc file to write',
dest='rc_file')
parser.add_argument('--first-resource-id',
type=int,
required=True,
help='first id for the generated string resources',
dest='first_resource_id')
parser.add_argument('--extract-datafile',
help='the python file execute that will define the '
'specific strings to extract from the source .grd file.'
'The module should contain a global array STRING_IDS '
'that specifies which string IDs need to be extracted '
'(if no global member by that name exists, then all the '
'strings are extracted). It may also optionally contain '
'a dictionary MODE_SPECIFIC_STRINGS which defines the '
'mode-specific strings to use for a given brand that is '
'extracted.',
dest='extract_datafile')
return parser
def main():
parser = BuildArgumentParser()
args = parser.parse_args()
string_ids_to_extract = []
mode_specific_strings = {}
extract_datafile = args.extract_datafile
if extract_datafile:
datafile_locals = dict();
exec(open(extract_datafile).read(), globals(), datafile_locals)
if 'STRING_IDS' in datafile_locals:
string_ids_to_extract = datafile_locals['STRING_IDS']
if 'MODE_SPECIFIC_STRINGS' in datafile_locals:
if not string_ids_to_extract:
parser.error('MODE_SPECIFIC_STRINGS was specified in file ' +
extract_datafile + ' but there were no specific STRING_IDS '
'specified for extraction')
mode_specific_strings = datafile_locals['MODE_SPECIFIC_STRINGS']
brand = args.brand
if brand:
if not mode_specific_strings:
parser.error('A brand was specified (' + brand + ') but no mode '
'specific strings were given.')
valid_brands = [b for b in
next(iter(mode_specific_strings.values())).keys()]
if not brand in valid_brands:
parser.error('A brand was specified (' + brand + ') but it is not '
'a valid brand [' + ', '.join(valid_brands) + '].')
elif mode_specific_strings:
parser.error('MODE_SPECIFIC_STRINGS were specified but no brand was '
'given.')
grd_files = args.input_grd_files
xtb_relative_paths = args.input_xtb_relative_paths
if len(grd_files) != len(xtb_relative_paths):
parser.error('Mismatch in number of grd files ({}) and xtb relative '
'paths ({})'.format(len(grd_files), len(xtb_relative_paths)))
inputs = zip(grd_files, xtb_relative_paths)
StringRcMaker(inputs, args.expected_xtb_input_files, args.header_file,
args.rc_file, brand, args.first_resource_id, string_ids_to_extract,
mode_specific_strings).MakeFiles()
return 0
if '__main__' == __name__:
sys.exit(main())
