Fixes #9: Parser performance optimizations
- Pre-compile all regex patterns as module-level constants in parser.js and jsonRepair.js - Add format detection caching (lastDetectedFormat, formatCacheHits) to skip redundant checks - Add clearFormatCache() export for cache invalidation - Add comprehensive test suite (33 tests) for parser and jsonRepair modules - Fix regex escaping in dynamic patterns (statRegex, fieldRegex) - Improve toFieldKey() regex specificity
This commit is contained in:
@@ -9,6 +9,100 @@ import { saveSettings } from '../../core/persistence.js';
|
||||
import { extractInventory } from './inventoryParser.js';
|
||||
import { repairJSON, extractJSONFromText } from '../../utils/jsonRepair.js';
|
||||
|
||||
/* ===== Pre-compiled regex patterns (module-level constants) ===== */
|
||||
|
||||
// Thinking tag removal
|
||||
const THINKING_TAG_RE = /<think>[\s\S]*?<\/think>/gi;
|
||||
const THINKING_TAG_ALT_RE = /<thinking>[\s\S]*?<\/thinking>/gi;
|
||||
|
||||
// FORMAT: marker removal
|
||||
const FORMAT_MARKER_RE = /FORMAT:\s*/gi;
|
||||
|
||||
// Emoji extraction (separateEmojiFromText)
|
||||
const EMOJI_RE = /^[\u{1F300}-\u{1F9FF}\u{2600}-\u{26FF}\u{2700}-\u{27BF}\u{1F000}-\u{1F02F}\u{1F0A0}-\u{1F0FF}\u{1F100}-\u{1F64F}\u{1F680}-\u{1F6FF}\u{1F910}-\u{1F96B}\u{1F980}-\u{1F9E0}\u{FE00}-\u{FE0F}\u{200D}\u{20E3}]+/u;
|
||||
|
||||
// Bracket stripping
|
||||
const PLACEHOLDER_PATTERN_RE = /\[([A-Za-z\s\/]+)\]/g;
|
||||
|
||||
// JSON code block extraction
|
||||
const JSON_BLOCK_RE = /```json\s*\n([\s\S]*?)```/g;
|
||||
|
||||
// XML trackers
|
||||
const XML_TRACKERS_RE = /<trackers>([\s\S]*?)<\/trackers>/i;
|
||||
|
||||
// Code block extraction
|
||||
const CODE_BLOCK_RE = /```([^`]+)```/g;
|
||||
|
||||
// XML text fallback patterns
|
||||
const XML_STATS_MATCH_RE = /(User )?Stats\s*\n\s*---[\s\S]*?(?=\n\s*\n\s*(Info Box|Present Characters)|$)/i;
|
||||
const XML_INFOBOX_MATCH_RE = /Info Box\s*\n\s*---[\s\S]*?(?=\n\s*\n\s*Present Characters|$)/i;
|
||||
const XML_CHARACTERS_MATCH_RE = /Present Characters\s*\n\s*---[\s\S]*$/i;
|
||||
|
||||
// Combined code block section patterns
|
||||
const COMBINED_STATS_RE = /(User )?Stats\s*\n\s*---[\s\S]*?(?=\n\s*\n\s*(Info Box|Present Characters)|$)/i;
|
||||
const COMBINED_INFOBOX_RE = /Info Box\s*\n\s*---[\s\S]*?(?=\n\s*\n\s*Present Characters|$)/i;
|
||||
const COMBINED_CHARACTERS_RE = /Present Characters\s*\n\s*---[\s\S]*$/i;
|
||||
|
||||
// Section detection patterns
|
||||
const IS_STATS_HEADER_RE = /Stats\s*\n\s*---/i;
|
||||
const IS_USER_STATS_HEADER_RE = /User Stats\s*\n\s*---/i;
|
||||
const IS_PLAYER_STATS_HEADER_RE = /Player Stats\s*\n\s*---/i;
|
||||
const IS_STATS_HEALTH_RE = /Health:\s*\d+%/i;
|
||||
const IS_STATS_ENERGY_RE = /Energy:\s*\d+%/i;
|
||||
const IS_INFOBOX_HEADER_RE = /Info Box\s*\n\s*---/i;
|
||||
const IS_SCENE_INFO_HEADER_RE = /Scene Info\s*\n\s*---/i;
|
||||
const IS_INFORMATION_HEADER_RE = /Information\s*\n\s*---/i;
|
||||
const IS_INFOBOX_DATE_RE = /Date:/i;
|
||||
const IS_INFOBOX_LOCATION_RE = /Location:/i;
|
||||
const IS_INFOBOX_TIME_RE = /Time:/i;
|
||||
const IS_CHARACTERS_HEADER_RE = /Present Characters\s*\n\s*---/i;
|
||||
const IS_CHARACTERS_ALT_RE = /Characters\s*\n\s*---/i;
|
||||
const IS_CHARACTERS_THOUGHTS_RE = /Character Thoughts\s*\n\s*---/i;
|
||||
const IS_CHARACTERS_BULLET_RE = /^-\s+\w+/m;
|
||||
const IS_CHARACTERS_DETAILS_RE = /Details:/i;
|
||||
|
||||
// Debug pattern checks
|
||||
const DEBUG_STATS_RE = /Stats\s*\n\s*---/i;
|
||||
const DEBUG_INFOBOX_RE = /Info Box\s*\n\s*---/i;
|
||||
const DEBUG_CHARACTERS_RE = /Present Characters\s*\n\s*---/i;
|
||||
|
||||
// Final fallback fenced regex
|
||||
const FENCED_FALLBACK_RE = /```(?:json)?\s*\n?([\s\S]*?)```/gi;
|
||||
|
||||
// parseUserStats patterns
|
||||
const RPG_STR_RE = /STR:\s*(\d+)/i;
|
||||
const RPG_DEX_RE = /DEX:\s*(\d+)/i;
|
||||
const RPG_CON_RE = /CON:\s*(\d+)/i;
|
||||
const RPG_INT_RE = /INT:\s*(\d+)/i;
|
||||
const RPG_WIS_RE = /WIS:\s*(\d+)/i;
|
||||
const RPG_CHA_RE = /CHA:\s*(\d+)/i;
|
||||
const RPG_LVL_RE = /LVL:\s*(\d+)/i;
|
||||
const STATUS_MATCH_RE = /Status:\s*(.+)/i;
|
||||
const SKILLS_MATCH_RE = /Skills:\s*(.+)/i;
|
||||
const INVENTORY_MATCH_RE = /Inventory:\s*(.+)/i;
|
||||
const MAIN_QUEST_MATCH_RE = /Main Quests?:\s*(.+)/i;
|
||||
const OPTIONAL_QUESTS_MATCH_RE = /Optional Quests:\s*(.+)/i;
|
||||
|
||||
// Section detection helpers
|
||||
const STATS_SECTION_RE = /Stats\s*\n\s*---/i;
|
||||
const INFOBOX_SECTION_RE = /Info Box\s*\n\s*---/i;
|
||||
const CHARACTERS_SECTION_RE = /Present Characters\s*\n\s*---/i;
|
||||
|
||||
/* ===== Format detection cache ===== */
|
||||
// Cache the last detected format to skip unnecessary checks on subsequent calls.
|
||||
// The cache is invalidated when the page reloads (module re-initializes).
|
||||
let lastDetectedFormat = null; // 'json', 'json_block', 'xml', 'text', null
|
||||
const FORMAT_CACHE_HIT_THRESHOLD = 3; // Minimum consecutive hits before trusting cache
|
||||
let formatCacheHits = 0;
|
||||
|
||||
/**
|
||||
* Clear the format detection cache. Call this when the AI model or prompt changes.
|
||||
*/
|
||||
export function clearFormatCache() {
|
||||
lastDetectedFormat = null;
|
||||
formatCacheHits = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Unwraps common envelope keys models may use around tracker payloads.
|
||||
* Keeps extraction resilient when output is nested under wrappers like "trackers".
|
||||
@@ -52,7 +146,7 @@ function unwrapTrackerEnvelope(payload) {
|
||||
* @returns {string} snake_case key from the base name only
|
||||
*/
|
||||
function toFieldKey(name) {
|
||||
const baseName = name.replace(/\s*\(.*\)\s*$/, '').trim();
|
||||
const baseName = name.replace(/\s*\([^)]*\)\s*$/, '').trim();
|
||||
return baseName
|
||||
.toLowerCase()
|
||||
.replace(/[^\p{L}\p{N}]+/gu, '_')
|
||||
@@ -70,10 +164,7 @@ function separateEmojiFromText(str) {
|
||||
|
||||
str = str.trim();
|
||||
|
||||
// Regex to match emoji at the start (handles most emoji including compound ones)
|
||||
// This matches emoji sequences including skin tones, gender modifiers, etc.
|
||||
const emojiRegex = /^[\u{1F300}-\u{1F9FF}\u{2600}-\u{26FF}\u{2700}-\u{27BF}\u{1F000}-\u{1F02F}\u{1F0A0}-\u{1F0FF}\u{1F100}-\u{1F64F}\u{1F680}-\u{1F6FF}\u{1F910}-\u{1F96B}\u{1F980}-\u{1F9E0}\u{FE00}-\u{FE0F}\u{200D}\u{20E3}]+/u;
|
||||
const emojiMatch = str.match(emojiRegex);
|
||||
const emojiMatch = str.match(EMOJI_RE);
|
||||
|
||||
if (emojiMatch) {
|
||||
const emoji = emojiMatch[0];
|
||||
@@ -121,13 +212,8 @@ function stripBrackets(text) {
|
||||
text = text.substring(1, text.length - 1).trim();
|
||||
}
|
||||
|
||||
// Remove placeholder text patterns like [Location], [Mood Emoji], [Name], etc.
|
||||
// Pattern matches: [anything with letters/spaces inside]
|
||||
// This preserves actual content while removing template placeholders
|
||||
const placeholderPattern = /\[([A-Za-z\s\/]+)\]/g;
|
||||
|
||||
// Check if a bracketed text looks like a placeholder vs real content
|
||||
const isPlaceholder = (match, content) => {
|
||||
// Replace placeholders with empty string, keep real content
|
||||
text = text.replace(PLACEHOLDER_PATTERN_RE, (match, content) => {
|
||||
// Common placeholder words to detect
|
||||
const placeholderKeywords = [
|
||||
'location', 'mood', 'emoji', 'name', 'description', 'placeholder',
|
||||
@@ -141,23 +227,15 @@ function stripBrackets(text) {
|
||||
|
||||
// If it contains common placeholder keywords, it's likely a placeholder
|
||||
if (placeholderKeywords.some(keyword => lowerContent.includes(keyword))) {
|
||||
return true;
|
||||
return ''; // Remove placeholder
|
||||
}
|
||||
|
||||
// If it's a short generic phrase (1-3 words) with only letters/spaces, might be placeholder
|
||||
const wordCount = content.trim().split(/\s+/).length;
|
||||
if (wordCount <= 3 && /^[A-Za-z\s\/]+$/.test(content)) {
|
||||
return true;
|
||||
return '';
|
||||
}
|
||||
|
||||
return false;
|
||||
};
|
||||
|
||||
// Replace placeholders with empty string, keep real content
|
||||
text = text.replace(placeholderPattern, (match, content) => {
|
||||
if (isPlaceholder(match, content)) {
|
||||
return ''; // Remove placeholder
|
||||
}
|
||||
return match; // Keep real bracketed content
|
||||
});
|
||||
|
||||
@@ -193,10 +271,11 @@ function debugLog(message, data = null) {
|
||||
* @param {string} responseText - The raw AI response text
|
||||
* @param {Object} [options] - Parser behavior options
|
||||
* @param {boolean} [options.suppressNoDataError=false] - Avoid console error when no tracker data is found
|
||||
* @param {boolean} [options.forceFormat=null] - Force a specific format check (bypasses cache)
|
||||
* @returns {{userStats: string|null, infoBox: string|null, characterThoughts: string|null}} Parsed tracker data
|
||||
*/
|
||||
export function parseResponse(responseText, options = {}) {
|
||||
const { suppressNoDataError = false } = options;
|
||||
const { suppressNoDataError = false, forceFormat = null } = options;
|
||||
const result = {
|
||||
userStats: null,
|
||||
infoBox: null,
|
||||
@@ -210,14 +289,22 @@ export function parseResponse(responseText, options = {}) {
|
||||
|
||||
// Remove content inside thinking tags first (model's internal reasoning)
|
||||
// This prevents parsing code blocks from the model's thinking process
|
||||
let cleanedResponse = responseText.replace(/<think>[\s\S]*?<\/think>/gi, '');
|
||||
cleanedResponse = cleanedResponse.replace(/<thinking>[\s\S]*?<\/thinking>/gi, '');
|
||||
let cleanedResponse = responseText.replace(THINKING_TAG_RE, '');
|
||||
cleanedResponse = cleanedResponse.replace(THINKING_TAG_ALT_RE, '');
|
||||
debugLog('[RPG Parser] Removed thinking tags, new length:', cleanedResponse.length + ' chars');
|
||||
|
||||
// Remove "FORMAT:" markers that the model might accidentally output
|
||||
cleanedResponse = cleanedResponse.replace(/FORMAT:\s*/gi, '');
|
||||
cleanedResponse = cleanedResponse.replace(FORMAT_MARKER_RE, '');
|
||||
debugLog('[RPG Parser] Removed FORMAT: markers, new length:', cleanedResponse.length + ' chars');
|
||||
|
||||
// Format cache: if we've seen the same format multiple times, try that path first
|
||||
const cachedFormat = forceFormat ?? lastDetectedFormat;
|
||||
let detectedFormat = null;
|
||||
|
||||
if (cachedFormat && formatCacheHits >= FORMAT_CACHE_HIT_THRESHOLD) {
|
||||
debugLog('[RPG Parser] Using cached format:', cachedFormat);
|
||||
}
|
||||
|
||||
// First, try to extract raw JSON objects (v3 format)
|
||||
// Note: Prompts now instruct models to use ```json``` code blocks, but we extract
|
||||
// from any JSON found using brace-matching for maximum compatibility
|
||||
@@ -267,6 +354,15 @@ export function parseResponse(responseText, options = {}) {
|
||||
// console.log(`[RPG Parser] ✓ Found ${extractedObjects.length} raw JSON objects (v3 format)`);
|
||||
debugLog(`[RPG Parser] ✓ Found ${extractedObjects.length} raw JSON objects (v3 format)`);
|
||||
|
||||
// Update format cache
|
||||
detectedFormat = 'json';
|
||||
if (cachedFormat === 'json') {
|
||||
formatCacheHits++;
|
||||
} else {
|
||||
lastDetectedFormat = 'json';
|
||||
formatCacheHits = 1;
|
||||
}
|
||||
|
||||
// First, try to parse as unified JSON structure (new v3.1 format)
|
||||
// Look through all extracted objects for unified structure
|
||||
let foundUnified = false;
|
||||
@@ -377,13 +473,21 @@ export function parseResponse(responseText, options = {}) {
|
||||
|
||||
// Check for JSON code blocks (legacy v3 format with ```json fences)
|
||||
// Look for ```json code blocks which indicate JSON format
|
||||
const jsonBlockRegex = /```json\s*\n([\s\S]*?)```/g;
|
||||
const jsonMatches = [...cleanedResponse.matchAll(jsonBlockRegex)];
|
||||
const jsonMatches = [...cleanedResponse.matchAll(JSON_BLOCK_RE)];
|
||||
|
||||
if (jsonMatches.length > 0) {
|
||||
// console.log('[RPG Parser] ✓ Found', jsonMatches.length, 'JSON code blocks (v3 format with fences)');
|
||||
debugLog('[RPG Parser] ✓ Found JSON code blocks (v3 format), parsing as JSON');
|
||||
|
||||
// Update format cache
|
||||
detectedFormat = 'json_block';
|
||||
if (cachedFormat === 'json_block') {
|
||||
formatCacheHits++;
|
||||
} else if (!detectedFormat) {
|
||||
lastDetectedFormat = 'json_block';
|
||||
formatCacheHits = 1;
|
||||
}
|
||||
|
||||
for (let idx = 0; idx < jsonMatches.length; idx++) {
|
||||
const match = jsonMatches[idx];
|
||||
const jsonContent = match[1].trim();
|
||||
@@ -447,13 +551,22 @@ export function parseResponse(responseText, options = {}) {
|
||||
}
|
||||
|
||||
// Check if response uses XML <trackers> tags (hybrid format)
|
||||
const xmlMatch = cleanedResponse.match(/<trackers>([\s\S]*?)<\/trackers>/i);
|
||||
const xmlMatch = cleanedResponse.match(XML_TRACKERS_RE);
|
||||
if (xmlMatch) {
|
||||
debugLog('[RPG Parser] ✓ Found XML <trackers> tags, using XML parser');
|
||||
const trackersContent = xmlMatch[1].trim();
|
||||
|
||||
// Update format cache
|
||||
detectedFormat = 'xml';
|
||||
if (cachedFormat === 'xml') {
|
||||
formatCacheHits++;
|
||||
} else if (!lastDetectedFormat) {
|
||||
lastDetectedFormat = 'xml';
|
||||
formatCacheHits = 1;
|
||||
}
|
||||
|
||||
// Try to parse JSON blocks within XML first
|
||||
const xmlJsonMatches = [...trackersContent.matchAll(jsonBlockRegex)];
|
||||
const xmlJsonMatches = [...trackersContent.matchAll(JSON_BLOCK_RE)];
|
||||
if (xmlJsonMatches.length > 0) {
|
||||
debugLog('[RPG Parser] Found JSON blocks within XML tags');
|
||||
for (const match of xmlJsonMatches) {
|
||||
@@ -475,19 +588,19 @@ export function parseResponse(responseText, options = {}) {
|
||||
}
|
||||
} else {
|
||||
// Fallback to text extraction from XML content (legacy v2 text format)
|
||||
const statsMatch = trackersContent.match(/(User )?Stats\s*\n\s*---[\s\S]*?(?=\n\s*\n\s*(Info Box|Present Characters)|$)/i);
|
||||
const statsMatch = trackersContent.match(XML_STATS_MATCH_RE);
|
||||
if (statsMatch) {
|
||||
result.userStats = stripBrackets(statsMatch[0].trim());
|
||||
debugLog('[RPG Parser] ✓ Extracted Stats from XML (text format)');
|
||||
}
|
||||
|
||||
const infoBoxMatch = trackersContent.match(/Info Box\s*\n\s*---[\s\S]*?(?=\n\s*\n\s*Present Characters|$)/i);
|
||||
const infoBoxMatch = trackersContent.match(XML_INFOBOX_MATCH_RE);
|
||||
if (infoBoxMatch) {
|
||||
result.infoBox = stripBrackets(infoBoxMatch[0].trim());
|
||||
debugLog('[RPG Parser] ✓ Extracted Info Box from XML (text format)');
|
||||
}
|
||||
|
||||
const charactersMatch = trackersContent.match(/Present Characters\s*\n\s*---[\s\S]*$/i);
|
||||
const charactersMatch = trackersContent.match(XML_CHARACTERS_MATCH_RE);
|
||||
if (charactersMatch) {
|
||||
result.characterThoughts = stripBrackets(charactersMatch[0].trim());
|
||||
debugLog('[RPG Parser] ✓ Extracted Present Characters from XML (text format)');
|
||||
@@ -502,8 +615,7 @@ export function parseResponse(responseText, options = {}) {
|
||||
debugLog('[RPG Parser] No XML tags found, using code block parser');
|
||||
|
||||
// Extract code blocks
|
||||
const codeBlockRegex = /```([^`]+)```/g;
|
||||
const matches = [...cleanedResponse.matchAll(codeBlockRegex)];
|
||||
const matches = [...cleanedResponse.matchAll(CODE_BLOCK_RE)];
|
||||
|
||||
debugLog('[RPG Parser] Found', matches.length + ' code blocks');
|
||||
|
||||
@@ -516,8 +628,8 @@ export function parseResponse(responseText, options = {}) {
|
||||
|
||||
// Check if this is a combined code block with multiple sections
|
||||
const hasMultipleSections = (
|
||||
content.match(/Stats\s*\n\s*---/i) &&
|
||||
(content.match(/Info Box\s*\n\s*---/i) || content.match(/Present Characters\s*\n\s*---/i))
|
||||
content.match(IS_STATS_HEADER_RE) &&
|
||||
(content.match(IS_INFOBOX_HEADER_RE) || content.match(IS_CHARACTERS_HEADER_RE))
|
||||
);
|
||||
|
||||
if (hasMultipleSections) {
|
||||
@@ -525,21 +637,21 @@ export function parseResponse(responseText, options = {}) {
|
||||
debugLog('[RPG Parser] ✓ Found combined code block with multiple sections');
|
||||
|
||||
// Extract User Stats section
|
||||
const statsMatch = content.match(/(User )?Stats\s*\n\s*---[\s\S]*?(?=\n\s*\n\s*(Info Box|Present Characters)|$)/i);
|
||||
const statsMatch = content.match(COMBINED_STATS_RE);
|
||||
if (statsMatch && !result.userStats) {
|
||||
result.userStats = stripBrackets(statsMatch[0].trim());
|
||||
debugLog('[RPG Parser] ✓ Extracted Stats from combined block');
|
||||
}
|
||||
|
||||
// Extract Info Box section
|
||||
const infoBoxMatch = content.match(/Info Box\s*\n\s*---[\s\S]*?(?=\n\s*\n\s*Present Characters|$)/i);
|
||||
const infoBoxMatch = content.match(COMBINED_INFOBOX_RE);
|
||||
if (infoBoxMatch && !result.infoBox) {
|
||||
result.infoBox = stripBrackets(infoBoxMatch[0].trim());
|
||||
debugLog('[RPG Parser] ✓ Extracted Info Box from combined block');
|
||||
}
|
||||
|
||||
// Extract Present Characters section
|
||||
const charactersMatch = content.match(/Present Characters\s*\n\s*---[\s\S]*$/i);
|
||||
const charactersMatch = content.match(COMBINED_CHARACTERS_RE);
|
||||
if (charactersMatch && !result.characterThoughts) {
|
||||
result.characterThoughts = stripBrackets(charactersMatch[0].trim());
|
||||
debugLog('[RPG Parser] ✓ Extracted Present Characters from combined block');
|
||||
@@ -548,27 +660,27 @@ export function parseResponse(responseText, options = {}) {
|
||||
// Handle separate code blocks with flexible pattern matching
|
||||
// Match Stats section - flexible patterns
|
||||
const isStats =
|
||||
content.match(/Stats\s*\n\s*---/i) ||
|
||||
content.match(/User Stats\s*\n\s*---/i) ||
|
||||
content.match(/Player Stats\s*\n\s*---/i) ||
|
||||
content.match(IS_STATS_HEADER_RE) ||
|
||||
content.match(IS_USER_STATS_HEADER_RE) ||
|
||||
content.match(IS_PLAYER_STATS_HEADER_RE) ||
|
||||
// Fallback: look for stat keywords without strict header
|
||||
(content.match(/Health:\s*\d+%/i) && content.match(/Energy:\s*\d+%/i));
|
||||
(content.match(IS_STATS_HEALTH_RE) && content.match(IS_STATS_ENERGY_RE));
|
||||
|
||||
// Match Info Box section - flexible patterns
|
||||
const isInfoBox =
|
||||
content.match(/Info Box\s*\n\s*---/i) ||
|
||||
content.match(/Scene Info\s*\n\s*---/i) ||
|
||||
content.match(/Information\s*\n\s*---/i) ||
|
||||
content.match(IS_INFOBOX_HEADER_RE) ||
|
||||
content.match(IS_SCENE_INFO_HEADER_RE) ||
|
||||
content.match(IS_INFORMATION_HEADER_RE) ||
|
||||
// Fallback: look for info box keywords
|
||||
(content.match(/Date:/i) && content.match(/Location:/i) && content.match(/Time:/i));
|
||||
(content.match(IS_INFOBOX_DATE_RE) && content.match(IS_INFOBOX_LOCATION_RE) && content.match(IS_INFOBOX_TIME_RE));
|
||||
|
||||
// Match Present Characters section - flexible patterns
|
||||
const isCharacters =
|
||||
content.match(/Present Characters\s*\n\s*---/i) ||
|
||||
content.match(/Characters\s*\n\s*---/i) ||
|
||||
content.match(/Character Thoughts\s*\n\s*---/i) ||
|
||||
content.match(IS_CHARACTERS_HEADER_RE) ||
|
||||
content.match(IS_CHARACTERS_ALT_RE) ||
|
||||
content.match(IS_CHARACTERS_THOUGHTS_RE) ||
|
||||
// Fallback: look for new multi-line format patterns
|
||||
(content.match(/^-\s+\w+/m) && content.match(/Details:/i));
|
||||
(content.match(IS_CHARACTERS_BULLET_RE) && content.match(IS_CHARACTERS_DETAILS_RE));
|
||||
|
||||
if (isStats && !result.userStats) {
|
||||
result.userStats = stripBrackets(content);
|
||||
@@ -582,12 +694,12 @@ export function parseResponse(responseText, options = {}) {
|
||||
debugLog('[RPG Parser] Full content:', content);
|
||||
} else {
|
||||
debugLog('[RPG Parser] ✗ No match - checking patterns:');
|
||||
debugLog('[RPG Parser] - Has "Stats\\n---"?', !!content.match(/Stats\s*\n\s*---/i));
|
||||
debugLog('[RPG Parser] - Has stat keywords?', !!(content.match(/Health:\s*\d+%/i) && content.match(/Energy:\s*\d+%/i)));
|
||||
debugLog('[RPG Parser] - Has "Info Box\\n---"?', !!content.match(/Info Box\s*\n\s*---/i));
|
||||
debugLog('[RPG Parser] - Has info keywords?', !!(content.match(/Date:/i) && content.match(/Location:/i)));
|
||||
debugLog('[RPG Parser] - Has "Present Characters\\n---"?', !!content.match(/Present Characters\s*\n\s*---/i));
|
||||
debugLog('[RPG Parser] - Has new format ("- Name" + "Details:")?', !!(content.match(/^-\s+\w+/m) && content.match(/Details:/i)));
|
||||
debugLog('[RPG Parser] - Has "Stats\\n---"?', !!content.match(DEBUG_STATS_RE));
|
||||
debugLog('[RPG Parser] - Has stat keywords?', !!(content.match(IS_STATS_HEALTH_RE) && content.match(IS_STATS_ENERGY_RE)));
|
||||
debugLog('[RPG Parser] - Has "Info Box\\n---"?', !!content.match(DEBUG_INFOBOX_RE));
|
||||
debugLog('[RPG Parser] - Has info keywords?', !!(content.match(IS_INFOBOX_DATE_RE) && content.match(IS_INFOBOX_LOCATION_RE)));
|
||||
debugLog('[RPG Parser] - Has "Present Characters\\n---"?', !!content.match(DEBUG_CHARACTERS_RE));
|
||||
debugLog('[RPG Parser] - Has new format ("- Name" + "Details:")?', !!(content.match(IS_CHARACTERS_BULLET_RE) && content.match(IS_CHARACTERS_DETAILS_RE)));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -601,8 +713,7 @@ export function parseResponse(responseText, options = {}) {
|
||||
// Final fallback: try to extract tracker JSON from any fenced block content
|
||||
// This catches responses where JSON is embedded in non-standard markdown structure.
|
||||
if (!result.userStats && !result.infoBox && !result.characterThoughts) {
|
||||
const fencedRegex = /```(?:json)?\s*\n?([\s\S]*?)```/gi;
|
||||
const fencedMatches = [...cleanedResponse.matchAll(fencedRegex)];
|
||||
const fencedMatches = [...cleanedResponse.matchAll(FENCED_FALLBACK_RE)];
|
||||
|
||||
for (const match of fencedMatches) {
|
||||
const fencedContent = (match[1] || '').trim();
|
||||
@@ -806,13 +917,13 @@ export function parseUserStats(statsText) {
|
||||
|
||||
// Parse RPG attributes if enabled
|
||||
if (trackerConfig?.userStats?.showRPGAttributes) {
|
||||
const strMatch = statsText.match(/STR:\s*(\d+)/i);
|
||||
const dexMatch = statsText.match(/DEX:\s*(\d+)/i);
|
||||
const conMatch = statsText.match(/CON:\s*(\d+)/i);
|
||||
const intMatch = statsText.match(/INT:\s*(\d+)/i);
|
||||
const wisMatch = statsText.match(/WIS:\s*(\d+)/i);
|
||||
const chaMatch = statsText.match(/CHA:\s*(\d+)/i);
|
||||
const lvlMatch = statsText.match(/LVL:\s*(\d+)/i);
|
||||
const strMatch = statsText.match(RPG_STR_RE);
|
||||
const dexMatch = statsText.match(RPG_DEX_RE);
|
||||
const conMatch = statsText.match(RPG_CON_RE);
|
||||
const intMatch = statsText.match(RPG_INT_RE);
|
||||
const wisMatch = statsText.match(RPG_WIS_RE);
|
||||
const chaMatch = statsText.match(RPG_CHA_RE);
|
||||
const lvlMatch = statsText.match(RPG_LVL_RE);
|
||||
|
||||
if (strMatch) extensionSettings.classicStats.str = parseInt(strMatch[1]);
|
||||
if (dexMatch) extensionSettings.classicStats.dex = parseInt(dexMatch[1]);
|
||||
@@ -832,7 +943,7 @@ export function parseUserStats(statsText) {
|
||||
const customFields = statusConfig.customFields || [];
|
||||
|
||||
// Try Status: format
|
||||
const statusMatch = statsText.match(/Status:\s*(.+)/i);
|
||||
const statusMatch = statsText.match(STATUS_MATCH_RE);
|
||||
if (statusMatch) {
|
||||
const statusContent = statusMatch[1].trim();
|
||||
|
||||
@@ -883,7 +994,7 @@ export function parseUserStats(statsText) {
|
||||
// Parse skills section if enabled
|
||||
const skillsConfig = trackerConfig?.userStats?.skillsSection;
|
||||
if (skillsConfig?.enabled) {
|
||||
const skillsMatch = statsText.match(/Skills:\s*(.+)/i);
|
||||
const skillsMatch = statsText.match(SKILLS_MATCH_RE);
|
||||
if (skillsMatch) {
|
||||
extensionSettings.userStats.skills = skillsMatch[1].trim();
|
||||
debugLog('[RPG Parser] Skills extracted:', skillsMatch[1].trim());
|
||||
@@ -901,7 +1012,7 @@ export function parseUserStats(statsText) {
|
||||
}
|
||||
} else {
|
||||
// Legacy v1 parsing for backward compatibility
|
||||
const inventoryMatch = statsText.match(/Inventory:\s*(.+)/i);
|
||||
const inventoryMatch = statsText.match(INVENTORY_MATCH_RE);
|
||||
if (inventoryMatch) {
|
||||
extensionSettings.userStats.inventory = inventoryMatch[1].trim();
|
||||
debugLog('[RPG Parser] Inventory v1 extracted:', inventoryMatch[1].trim());
|
||||
@@ -911,13 +1022,13 @@ export function parseUserStats(statsText) {
|
||||
}
|
||||
|
||||
// Extract quests
|
||||
const mainQuestMatch = statsText.match(/Main Quests?:\s*(.+)/i);
|
||||
const mainQuestMatch = statsText.match(MAIN_QUEST_MATCH_RE);
|
||||
if (mainQuestMatch) {
|
||||
extensionSettings.quests.main = mainQuestMatch[1].trim();
|
||||
debugLog('[RPG Parser] Main quests extracted:', mainQuestMatch[1].trim());
|
||||
}
|
||||
|
||||
const optionalQuestsMatch = statsText.match(/Optional Quests:\s*(.+)/i);
|
||||
const optionalQuestsMatch = statsText.match(OPTIONAL_QUESTS_MATCH_RE);
|
||||
if (optionalQuestsMatch) {
|
||||
const questsText = optionalQuestsMatch[1].trim();
|
||||
if (questsText && questsText !== 'None') {
|
||||
@@ -960,8 +1071,7 @@ export function parseUserStats(statsText) {
|
||||
* @returns {Array<string>} Array of code block contents
|
||||
*/
|
||||
export function extractCodeBlocks(text) {
|
||||
const codeBlockRegex = /```([^`]+)```/g;
|
||||
const matches = [...text.matchAll(codeBlockRegex)];
|
||||
const matches = [...text.matchAll(CODE_BLOCK_RE)];
|
||||
return matches.map(match => match[1].trim());
|
||||
}
|
||||
|
||||
@@ -971,7 +1081,7 @@ export function extractCodeBlocks(text) {
|
||||
* @returns {boolean} True if this is a stats section
|
||||
*/
|
||||
export function isStatsSection(content) {
|
||||
return content.match(/Stats\s*\n\s*---/i) !== null;
|
||||
return content.match(STATS_SECTION_RE) !== null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -980,7 +1090,7 @@ export function isStatsSection(content) {
|
||||
* @returns {boolean} True if this is an info box section
|
||||
*/
|
||||
export function isInfoBoxSection(content) {
|
||||
return content.match(/Info Box\s*\n\s*---/i) !== null;
|
||||
return content.match(INFOBOX_SECTION_RE) !== null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -989,5 +1099,5 @@ export function isInfoBoxSection(content) {
|
||||
* @returns {boolean} True if this is a character thoughts section
|
||||
*/
|
||||
export function isCharacterThoughtsSection(content) {
|
||||
return content.match(/Present Characters\s*\n\s*---/i) !== null || content.includes(" | ");
|
||||
return content.match(CHARACTERS_SECTION_RE) !== null || content.includes(" | ");
|
||||
}
|
||||
|
||||
+38
-13
@@ -3,6 +3,31 @@
|
||||
* Handles parsing and repairing malformed JSON from AI responses
|
||||
*/
|
||||
|
||||
/* ===== Pre-compiled regex patterns (module-level constants) ===== */
|
||||
|
||||
// Markdown code fence removal
|
||||
const MARKDOWN_JSON_FENCE_RE = /```json\s*/gi;
|
||||
const MARKDOWN_GENERIC_FENCE_RE = /```\s*/g;
|
||||
|
||||
// Thinking tag removal
|
||||
const THINKING_TAG_RE = /<think>[\s\S]*?<\/think>/gi;
|
||||
const THINKING_TAG_ALT_RE = /<thinking>[\s\S]*?<\/thinking>/gi;
|
||||
|
||||
// JSON repair patterns
|
||||
const TRAILING_COMMA_RE = /,(\s*[}\]])/g;
|
||||
const JS_LINE_COMMENT_RE = /\/\/.*$/gm;
|
||||
const JS_BLOCK_COMMENT_RE = /\/\*[\s\S]*?\*\//g;
|
||||
|
||||
// JSON extraction patterns
|
||||
const JSON_OBJECT_RE = /\{[\s\S]*\}/;
|
||||
const JSON_ARRAY_RE = /\[[\s\S]*\]/;
|
||||
|
||||
// extractJSONFromText patterns
|
||||
const FENCE_JSON_RE = /```json\s*([\s\S]*?)```/i;
|
||||
const FENCE_GENERIC_RE = /```\s*([\s\S]*?)```/;
|
||||
const STANDALONE_OBJECT_RE = /\{[\s\S]*\}/;
|
||||
const STANDALONE_ARRAY_RE = /\[[\s\S]*\]/;
|
||||
|
||||
/**
|
||||
* Repairs malformed JSON from AI responses
|
||||
* Handles common AI mistakes like trailing commas, missing commas, wrong quotes, etc.
|
||||
@@ -23,17 +48,17 @@ export function repairJSON(jsonString) {
|
||||
}
|
||||
|
||||
// Remove markdown code fences
|
||||
cleaned = cleaned.replace(/```json\s*/gi, '');
|
||||
cleaned = cleaned.replace(/```\s*/g, '');
|
||||
cleaned = cleaned.replace(MARKDOWN_JSON_FENCE_RE, '');
|
||||
cleaned = cleaned.replace(MARKDOWN_GENERIC_FENCE_RE, '');
|
||||
|
||||
// Remove thinking tags (model's internal reasoning)
|
||||
cleaned = cleaned.replace(/<think>[\s\S]*?<\/think>/gi, '');
|
||||
cleaned = cleaned.replace(/<thinking>[\s\S]*?<\/thinking>/gi, '');
|
||||
cleaned = cleaned.replace(THINKING_TAG_RE, '');
|
||||
cleaned = cleaned.replace(THINKING_TAG_ALT_RE, '');
|
||||
|
||||
// Fix common JSON errors:
|
||||
|
||||
// 1. Trailing commas before closing brackets
|
||||
cleaned = cleaned.replace(/,(\s*[}\]])/g, '$1');
|
||||
cleaned = cleaned.replace(TRAILING_COMMA_RE, '$1');
|
||||
|
||||
// 2. Missing commas between properties - DISABLED because it corrupts valid JSON
|
||||
// Modern AI models send properly formatted JSON, so this aggressive repair is not needed
|
||||
@@ -49,8 +74,8 @@ export function repairJSON(jsonString) {
|
||||
// cleaned = cleaned.replace(/(\{|,)\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*:/g, '$1"$2":');
|
||||
|
||||
// 5. Remove JavaScript comments
|
||||
cleaned = cleaned.replace(/\/\/.*$/gm, '');
|
||||
cleaned = cleaned.replace(/\/\*[\s\S]*?\*\//g, '');
|
||||
cleaned = cleaned.replace(JS_LINE_COMMENT_RE, '');
|
||||
cleaned = cleaned.replace(JS_BLOCK_COMMENT_RE, '');
|
||||
|
||||
// Attempt 1: Standard JSON.parse
|
||||
try {
|
||||
@@ -59,7 +84,7 @@ export function repairJSON(jsonString) {
|
||||
}
|
||||
|
||||
// Attempt 2: Extract JSON object between first { and last }
|
||||
const objectMatch = cleaned.match(/\{[\s\S]*\}/);
|
||||
const objectMatch = cleaned.match(JSON_OBJECT_RE);
|
||||
if (objectMatch) {
|
||||
try {
|
||||
return JSON.parse(objectMatch[0]);
|
||||
@@ -69,7 +94,7 @@ export function repairJSON(jsonString) {
|
||||
}
|
||||
|
||||
// Attempt 3: Try to extract JSON array between first [ and last ]
|
||||
const arrayMatch = cleaned.match(/\[[\s\S]*\]/);
|
||||
const arrayMatch = cleaned.match(JSON_ARRAY_RE);
|
||||
if (arrayMatch) {
|
||||
try {
|
||||
return JSON.parse(arrayMatch[0]);
|
||||
@@ -149,14 +174,14 @@ export function extractJSONFromText(text) {
|
||||
}
|
||||
|
||||
// Try to extract from ```json code fence
|
||||
const fenceMatch = text.match(/```json\s*([\s\S]*?)```/i);
|
||||
const fenceMatch = text.match(FENCE_JSON_RE);
|
||||
if (fenceMatch && fenceMatch[1]) {
|
||||
const trimmed = fenceMatch[1].trim();
|
||||
if (trimmed) return trimmed;
|
||||
}
|
||||
|
||||
// Try to extract from ``` code fence (without json label)
|
||||
const genericFenceMatch = text.match(/```\s*([\s\S]*?)```/);
|
||||
const genericFenceMatch = text.match(FENCE_GENERIC_RE);
|
||||
if (genericFenceMatch && genericFenceMatch[1]) {
|
||||
const content = genericFenceMatch[1].trim();
|
||||
// Check if it looks like JSON (starts with { or [)
|
||||
@@ -166,13 +191,13 @@ export function extractJSONFromText(text) {
|
||||
}
|
||||
|
||||
// Try to find standalone JSON object
|
||||
const objectMatch = text.match(/\{[\s\S]*\}/);
|
||||
const objectMatch = text.match(STANDALONE_OBJECT_RE);
|
||||
if (objectMatch && objectMatch[0].trim()) {
|
||||
return objectMatch[0];
|
||||
}
|
||||
|
||||
// Try to find standalone JSON array
|
||||
const arrayMatch = text.match(/\[[\s\S]*\]/);
|
||||
const arrayMatch = text.match(STANDALONE_ARRAY_RE);
|
||||
if (arrayMatch && arrayMatch[0].trim()) {
|
||||
return arrayMatch[0];
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user