* Copyright (c) 2023 Huawei Technologies Co.,Ltd.
*
* openInula is licensed under Mulan PSL v2.
* You can use this software according to the terms and conditions of the Mulan PSL v2.
* You may obtain a copy of Mulan PSL v2 at:
*
* http://license.coscl.org.cn/MulanPSL2
*
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PSL v2 for more details.
*/
import Lexer from './Lexer';
import { mappingRule } from './mappingRule';
import ruleUtils from '../utils/parseRuleUtils';
import { RawToken } from '../types/types';
import { STATE_GROUP_START_INDEX, GLOBAL_FLAG, STICKY_FLAG, UNICODE_FLAG, VERTICAL_LINE } from '../constants';
const defaultErrorRule = ruleUtils.getRuleOptions('error', { lineBreaks: true, shouldThrow: true });
function parseRules(rules: Record<string, any>, hasStates: boolean): Record<string, object> {
let errorRule: Record<string, object> | null = null;
const fast: Record<string, unknown> = {};
let enableFast = true;
let unicodeFlag: boolean | null = null;
const groups: Record<string, any>[] = [];
const parts: string[] = [];
enableFast = isExistsFallback(rules, enableFast);
for (let i = 0; i < rules.length; i++) {
const options = rules[i];
if (options.include) {
throw new Error('Inheritance is not allowed in stateless lexers!');
}
errorRule = isOptionsErrorOrFallback(options, errorRule);
const match = options.match.slice();
if (enableFast) {
processFast(match, fast, options);
}
if (options.pop || options.push || options.next) {
checkStateOptions(hasStates, options);
}
if (match.length === 0) {
continue;
}
enableFast = false;
groups.push(options);
unicodeFlag = checkUnicode(match, unicodeFlag, options);
const pat = ruleUtils.getRegUnion(match.map(ruleUtils.getReg));
const regexp = new RegExp(pat);
if (regexp.test('')) {
throw new Error('The regex matched the empty string!');
}
const groupCount = ruleUtils.getRegGroups(pat);
if (groupCount > 0) {
throw new Error('The regular expression uses capture groups, use (?: … ) instead!');
}
if (!options.lineBreaks && regexp.test('\n')) {
throw new Error('The matching rule must contain lineBreaks.');
}
parts.push(ruleUtils.getRegCapture(pat));
}
const fallbackRule = errorRule && errorRule.fallback;
let flags = ruleUtils.checkSticky() && !fallbackRule ? STICKY_FLAG : GLOBAL_FLAG;
const suffix = ruleUtils.checkSticky() || fallbackRule ? '' : VERTICAL_LINE;
if (unicodeFlag === true) {
flags += UNICODE_FLAG;
}
const combined = new RegExp(ruleUtils.getRegUnion(parts) + suffix, flags);
return {
regexp: combined,
groups: groups,
fast: fast,
error: errorRule || defaultErrorRule,
};
}
export function checkStateGroup(group: Record<string, any>, name: string, mappingRules: Record<string, object>) {
const state = group && (group.push || group.next);
if (state && !mappingRules[state]) {
throw new Error('The state is missing.');
}
if (group && group.pop && +group.pop !== STATE_GROUP_START_INDEX) {
throw new Error('The value of pop must be 1.');
}
}
function parseMappingRule(mappingRule: Record<string, object>, startState?: string): Lexer<RawToken> {
const keys = Object.getOwnPropertyNames(mappingRule);
if (!startState) {
startState = keys[0];
}
const ruleMap = keys.reduce((map, key) => {
map[key] = ruleUtils.getRules(mappingRule[key]);
return map;
}, {});
for (let i = 0; i < keys.length; i++) {
const key = keys[i];
const rules = ruleMap[key];
const included = {};
for (let j = 0; j < rules.length; j++) {
const rule = rules[j];
if (!rule.include) {
continue;
}
const splice = [j, STATE_GROUP_START_INDEX];
if (rule.include !== key && !included[rule.include]) {
included[rule.include] = true;
const newRules = ruleMap[rule.include];
if (!newRules) {
throw new Error('Cannot contain a state that does not exist!');
}
newRules.forEach(newRule => {
if (!rules.includes(newRule)) {
splice.push(newRule);
}
});
}
rules.splice.apply(rules, splice);
j--;
}
}
const mappingAllRules = {};
keys.forEach(key => {
mappingAllRules[key] = parseRules(ruleMap[key], true);
});
keys.forEach(name => {
const state = mappingAllRules[name];
const groups = state.groups;
groups.forEach(group => {
checkStateGroup(group, name, mappingAllRules);
});
const fastKeys = Object.getOwnPropertyNames(state.fast);
fastKeys.forEach(fastKey => {
checkStateGroup(state.fast[fastKey], name, mappingAllRules);
});
});
return new Lexer(mappingAllRules, startState);
}
* 快速匹配模式
* @param match
* @param fast
* @param options
*/
function processFast(match: Record<string, any>, fast: Record<string, unknown> = {}, options: Record<string, object>) {
while (match.length && typeof match[0] === 'string' && match[0].length === 1) {
const word = match.shift();
fast[word.charCodeAt(0)] = options;
}
}
* 用以处理错误逻辑
* @param options 操作属性
* @param errorRule 错误规则
*/
function handleErrorRule(options: Record<string, object>, errorRule: Record<string, object>) {
if (!options.fallback === !errorRule.fallback) {
throw new Error('errorRule can only set one!');
} else {
throw new Error('fallback and error cannot be set at the same time!');
}
}
* 用以检查message中是否包含Unicode
* @param match 匹配到的message
* @param unicodeFlag Unicode标志
* @param options 操作属性
*/
function checkUnicode(match: Record<string, any>, unicodeFlag: boolean | null, options: Record<string, any>) {
for (let j = 0; j < match.length; j++) {
const obj = match[j];
if (!ruleUtils.checkRegExp(obj)) {
continue;
}
if (unicodeFlag === null) {
unicodeFlag = obj.unicode;
} else {
if (unicodeFlag !== obj.unicode && options.fallback === false) {
throw new Error('If the /u flag is used, all!');
}
}
}
return unicodeFlag;
}
function checkStateOptions(hasStates: boolean, options: Record<string, any>) {
if (!hasStates) {
throw new Error('State toggle options are not allowed in stateless tokenizers!');
}
if (options.fallback) {
throw new Error('State toggle options are not allowed on fallback tokens!');
}
}
* 检查是否存在fallback属性,用以来判断快速匹配规则
* @param rules
* @param enableFast
*/
function isExistsFallback(rules: Record<string, any>, enableFast: boolean) {
for (let i = 0; i < rules.length; i++) {
if (rules[i].fallback) {
enableFast = false;
}
}
return enableFast;
}
function isOptionsErrorOrFallback(options: Record<string, object>, errorRule: Record<string, object> | null) {
if (options.error || options.fallback) {
if (errorRule) {
handleErrorRule(options, errorRule);
}
errorRule = options;
}
return errorRule;
}
export const lexer = parseMappingRule(mappingRule);
export default parseMappingRule;