feat: json format, et al.

src/systems/generation/parser.js

@@ -1,11 +1,14 @@
 /**
  * Parser Module
  * Handles parsing of AI responses to extract tracker data
+ * Supports both legacy text format and new JSON format
  */
 
 import { extensionSettings, FEATURE_FLAGS, addDebugLog } from '../../core/state.js';
-import { saveSettings } from '../../core/persistence.js';
+import { saveSettings, saveChatData } from '../../core/persistence.js';
 import { extractInventory } from './inventoryParser.js';
+import { validateTrackerData, mergeTrackerData } from '../../types/trackerData.js';
+import { handleItemRemoved } from '../rendering/skills.js';
 
 /**
  * Helper to separate emoji from text in a string
@@ -133,19 +136,296 @@ function debugLog(message, data = null) {
     }
 }
 
+/**
+ * Extracts JSON from a code block (handles ```json ... ``` format)
+ * @param {string} text - Text that may contain JSON code blocks
+ * @returns {Object|null} Parsed JSON object or null
+ */
+function extractJSONFromCodeBlock(text) {
+    if (!text) return null;
+    
+    // Match ```json ... ``` or ``` ... ``` blocks
+    const jsonBlockRegex = /```(?:json)?\s*([\s\S]*?)```/gi;
+    const matches = [...text.matchAll(jsonBlockRegex)];
+    
+    for (const match of matches) {
+        const content = match[1].trim();
+        // Check if content looks like JSON (starts with { or [)
+        if (content.startsWith('{') || content.startsWith('[')) {
+            try {
+                return JSON.parse(content);
+            } catch (e) {
+                debugLog('[RPG Parser] JSON parse failed:', e.message);
+                // Try to fix common JSON issues
+                const fixed = tryFixJSON(content);
+                if (fixed) return fixed;
+            }
+        }
+    }
+    
+    return null;
+}
+
+/**
+ * Attempts to fix common JSON formatting issues
+ * @param {string} jsonStr - Potentially malformed JSON string
+ * @returns {Object|null} Fixed JSON object or null
+ */
+function tryFixJSON(jsonStr) {
+    try {
+        // Remove trailing commas
+        let fixed = jsonStr.replace(/,(\s*[}\]])/g, '$1');
+        // Fix unquoted keys
+        fixed = fixed.replace(/([{,]\s*)(\w+)(\s*:)/g, '$1"$2"$3');
+        return JSON.parse(fixed);
+    } catch (e) {
+        return null;
+    }
+}
+
+/**
+ * Parses JSON tracker data and applies it to extension settings
+ * @param {Object} jsonData - Parsed JSON tracker data
+ * @returns {boolean} Whether parsing was successful
+ */
+export function parseJSONTrackerData(jsonData) {
+    debugLog('[RPG Parser] ==================== JSON PARSING ====================');
+    
+    const validation = validateTrackerData(jsonData);
+    if (!validation.valid) {
+        debugLog('[RPG Parser] JSON validation failed:', validation.errors);
+        return false;
+    }
+    
+    const trackerConfig = extensionSettings.trackerConfig;
+    
+    // Parse stats
+    if (jsonData.stats) {
+        debugLog('[RPG Parser] Parsing stats:', Object.keys(jsonData.stats));
+        const customStats = trackerConfig?.userStats?.customStats || [];
+        
+        for (const [statName, value] of Object.entries(jsonData.stats)) {
+            // Find matching stat in config
+            const statConfig = customStats.find(s => 
+                s.name.toLowerCase() === statName.toLowerCase()
+            );
+            if (statConfig && typeof value === 'number') {
+                // Store in userStats using the stat id
+                extensionSettings.userStats[statConfig.id] = Math.max(0, Math.min(100, value));
+                debugLog(`[RPG Parser] Stat ${statConfig.name}: ${value}%`);
+            }
+        }
+    }
+    
+    // Parse status
+    if (jsonData.status) {
+        if (jsonData.status.mood) {
+            extensionSettings.userStats.mood = jsonData.status.mood;
+            debugLog('[RPG Parser] Mood:', jsonData.status.mood);
+        }
+        if (jsonData.status.fields) {
+            // Store custom status fields
+            extensionSettings.userStats.conditions = Object.values(jsonData.status.fields).join(', ') || 'None';
+            debugLog('[RPG Parser] Status fields:', jsonData.status.fields);
+        }
+    }
+    
+    // Parse infoBox - normalize values and filter out null
+    if (jsonData.infoBox) {
+        const infoBox = {};
+        // Only copy non-null values
+        for (const [key, val] of Object.entries(jsonData.infoBox)) {
+            if (val !== null && val !== undefined && val !== 'null') {
+                infoBox[key] = val;
+            }
+        }
+        // Normalize recentEvents - LLM sometimes returns string instead of array
+        if (infoBox.recentEvents && typeof infoBox.recentEvents === 'string') {
+            infoBox.recentEvents = [infoBox.recentEvents];
+        } else if (!infoBox.recentEvents || !Array.isArray(infoBox.recentEvents)) {
+            infoBox.recentEvents = [];
+        }
+        // Filter out null/empty events from array
+        infoBox.recentEvents = infoBox.recentEvents.filter(e => e && e !== 'null');
+        extensionSettings.infoBoxData = infoBox;
+        debugLog('[RPG Parser] InfoBox:', Object.keys(infoBox));
+    }
+    
+    // Parse characters - store for UI rendering
+    if (jsonData.characters && Array.isArray(jsonData.characters)) {
+        extensionSettings.charactersData = jsonData.characters;
+        debugLog('[RPG Parser] Characters:', jsonData.characters.length);
+    }
+    
+    // Parse inventory (structured format)
+    // Handle LLM variations: empty objects {} should become empty arrays []
+    if (jsonData.inventory) {
+        const normalizeArray = (val) => {
+            if (Array.isArray(val)) return val;
+            if (val && typeof val === 'object' && Object.keys(val).length === 0) return [];
+            return [];
+        };
+        
+        // Get all item names from current inventory BEFORE updating
+        const getItemNamesFromInventory = (inv) => {
+            const names = new Set();
+            if (!inv) return names;
+            // From onPerson array
+            if (Array.isArray(inv.onPerson)) {
+                inv.onPerson.forEach(item => {
+                    const name = typeof item === 'string' ? item : item?.name;
+                    if (name) names.add(name.toLowerCase());
+                });
+            }
+            // From stored locations
+            if (inv.stored && typeof inv.stored === 'object') {
+                Object.values(inv.stored).forEach(items => {
+                    if (Array.isArray(items)) {
+                        items.forEach(item => {
+                            const name = typeof item === 'string' ? item : item?.name;
+                            if (name) names.add(name.toLowerCase());
+                        });
+                    }
+                });
+            }
+            // From assets
+            if (Array.isArray(inv.assets)) {
+                inv.assets.forEach(item => {
+                    const name = typeof item === 'string' ? item : item?.name;
+                    if (name) names.add(name.toLowerCase());
+                });
+            }
+            return names;
+        };
+        
+        const previousItemNames = getItemNamesFromInventory(extensionSettings.inventoryV3);
+        
+        extensionSettings.inventoryV3 = {
+            onPerson: normalizeArray(jsonData.inventory.onPerson || jsonData.inventory.items),
+            stored: jsonData.inventory.stored && typeof jsonData.inventory.stored === 'object' 
+                ? jsonData.inventory.stored : {},
+            assets: normalizeArray(jsonData.inventory.assets)
+        };
+        debugLog('[RPG Parser] Inventory parsed - onPerson:', extensionSettings.inventoryV3.onPerson.length);
+        
+        // Detect removed items and handle skill links
+        const newItemNames = getItemNamesFromInventory(extensionSettings.inventoryV3);
+        previousItemNames.forEach(itemName => {
+            if (!newItemNames.has(itemName)) {
+                debugLog('[RPG Parser] Item removed by LLM:', itemName);
+                // Handle item removal - will unlink or delete skills based on settings
+                handleItemRemoved(itemName);
+            }
+        });
+        
+        // Also update legacy inventory for backwards compatibility
+        const itemsToString = (items) => {
+            if (!items || items.length === 0) return 'None';
+            return items.map(i => i.name).join(', ');
+        };
+        extensionSettings.userStats.inventory = {
+            version: 2,
+            onPerson: itemsToString(extensionSettings.inventoryV3.onPerson),
+            stored: Object.fromEntries(
+                Object.entries(extensionSettings.inventoryV3.stored).map(([k, v]) => [k, itemsToString(v)])
+            ),
+            assets: itemsToString(extensionSettings.inventoryV3.assets)
+        };
+    }
+    
+    // Parse skills (structured format) - handle array/object/string variations
+    if (jsonData.skills && typeof jsonData.skills === 'object') {
+        // Normalize skills - each category should be an array
+        const normalizedSkills = {};
+        for (const [category, abilities] of Object.entries(jsonData.skills)) {
+            if (Array.isArray(abilities)) {
+                normalizedSkills[category] = abilities;
+            } else if (typeof abilities === 'string') {
+                // LLM returned string instead of array - split by comma
+                normalizedSkills[category] = abilities.split(',').map(a => ({ name: a.trim(), description: '' })).filter(a => a.name);
+            } else {
+                normalizedSkills[category] = [];
+            }
+        }
+        extensionSettings.skillsV2 = normalizedSkills;
+        debugLog('[RPG Parser] Skills parsed:', Object.keys(normalizedSkills));
+        
+        // Update legacy skills data for backwards compatibility
+        for (const [category, abilities] of Object.entries(normalizedSkills)) {
+            if (!extensionSettings.skillsData) extensionSettings.skillsData = {};
+            const names = abilities.map(a => typeof a === 'string' ? a : (a?.name || '')).filter(n => n);
+            extensionSettings.skillsData[category] = names.join(', ') || 'None';
+        }
+    }
+    
+    // Parse quests - handle different formats
+    if (jsonData.quests) {
+        // Normalize main quest - could be string, object with name, or null
+        let mainName = 'None';
+        let mainDesc = '';
+        if (jsonData.quests.main) {
+            if (typeof jsonData.quests.main === 'string') {
+                mainName = jsonData.quests.main;
+            } else if (jsonData.quests.main.name) {
+                mainName = jsonData.quests.main.name;
+                mainDesc = jsonData.quests.main.description || '';
+            }
+        }
+        
+        // Normalize optional quests - could be array of strings or objects
+        const optionalQuests = Array.isArray(jsonData.quests.optional) ? jsonData.quests.optional : [];
+        const optionalNames = optionalQuests.map(q => typeof q === 'string' ? q : (q?.name || '')).filter(n => n);
+        const optionalDescs = optionalQuests.map(q => typeof q === 'string' ? '' : (q?.description || ''));
+        
+        extensionSettings.quests = {
+            main: mainName,
+            mainDescription: mainDesc,
+            optional: optionalNames,
+            optionalDescriptions: optionalDescs
+        };
+        // Store structured quests too
+        extensionSettings.questsV2 = jsonData.quests;
+        debugLog('[RPG Parser] Quests - main:', mainName);
+    }
+    
+    saveSettings();
+    saveChatData();
+    
+    debugLog('[RPG Parser] JSON parsing complete');
+    debugLog('[RPG Parser] =======================================================');
+    
+    return true;
+}
+
+/**
+ * Main entry point for parsing responses - tries JSON first, falls back to text
+ * @param {string} responseText - The raw AI response
+ * @returns {boolean} Whether JSON parsing was successful
+ */
+export function tryParseJSONResponse(responseText) {
+    const jsonData = extractJSONFromCodeBlock(responseText);
+    if (jsonData) {
+        return parseJSONTrackerData(jsonData);
+    }
+    
+    debugLog('[RPG Parser] No valid JSON found, falling back to text parsing');
+    return false;
+}
+
 /**
  * Parses the model response to extract the different data sections.
  * Extracts tracker data from markdown code blocks in the AI response.
  * Handles both separate code blocks and combined code blocks gracefully.
  *
  * @param {string} responseText - The raw AI response text
- * @returns {{userStats: string|null, infoBox: string|null, characterThoughts: string|null}} Parsed tracker data
+ * @returns {{userStats: string|null, infoBox: string|null, characterThoughts: string|null, skills: string|null}} Parsed tracker data
  */
 export function parseResponse(responseText) {
     const result = {
         userStats: null,
         infoBox: null,
-        characterThoughts: null
+        characterThoughts: null,
+        skills: null
     };
 
     // DEBUG: Log full response for troubleshooting
@@ -213,7 +493,12 @@ export function parseResponse(responseText) {
                 (content.match(/Health:\s*\d+%/i) && content.match(/Energy:\s*\d+%/i)) ||
                 // Fallback: inventory-only or quests-only blocks (no stats header)
                 (content.match(/^(On Person:|Inventory:|Main Quests?:|Optional Quests:)/im) && 
-                 !content.match(/Info Box/i) && !content.match(/Present Characters/i));
+                 !content.match(/Info Box/i) && !content.match(/Present Characters/i) && !content.match(/Skills\s*\n\s*---/i));
+
+            // Match Skills section (separate from stats when showSkills is enabled)
+            const isSkills =
+                content.match(/Skills\s*\n\s*---/i) &&
+                !content.match(/Stats\s*\n\s*---/i); // Make sure it's not a combined block
 
             // Match Info Box section - flexible patterns
             const isInfoBox =
@@ -234,6 +519,9 @@ export function parseResponse(responseText) {
             if (isStats && !result.userStats) {
                 result.userStats = stripBrackets(content);
                 debugLog('[RPG Parser] ✓ Matched: Stats section');
+            } else if (isSkills && !result.skills) {
+                result.skills = stripBrackets(content);
+                debugLog('[RPG Parser] ✓ Matched: Skills section');
             } else if (isInfoBox && !result.infoBox) {
                 result.infoBox = stripBrackets(content);
                 debugLog('[RPG Parser] ✓ Matched: Info Box section');
@@ -249,12 +537,14 @@ export function parseResponse(responseText) {
                 debugLog('[RPG Parser]   - Has info keywords?', !!(content.match(/Date:/i) && content.match(/Location:/i)));
                 debugLog('[RPG Parser]   - Has "Present Characters\\n---"?', !!content.match(/Present Characters\s*\n\s*---/i));
                 debugLog('[RPG Parser]   - Has new format ("- Name" + "Details:")?', !!(content.match(/^-\s+\w+/m) && content.match(/Details:/i)));
+                debugLog('[RPG Parser]   - Has "Skills\\n---"?', !!content.match(/Skills\s*\n\s*---/i));
             }
         }
     }
 
     debugLog('[RPG Parser] ==================== PARSE RESULTS ====================');
     debugLog('[RPG Parser] Found Stats:', !!result.userStats);
+    debugLog('[RPG Parser] Found Skills:', !!result.skills);
     debugLog('[RPG Parser] Found Info Box:', !!result.infoBox);
     debugLog('[RPG Parser] Found Characters:', !!result.characterThoughts);
     debugLog('[RPG Parser] =======================================================');
@@ -425,6 +715,125 @@ export function parseUserStats(statsText) {
     }
 }
 
+/**
+ * Parses skills from the separate skills code block.
+ * Used when showSkills is enabled (skills appear in their own section).
+ * Format expected:
+ * Skills
+ * ---
+ * Combat: Sword Fighting, Shield Block, Parry
+ * Stealth: Lockpicking (via Lockpicks), Sneaking, Pickpocketing
+ * Magic: None
+ *
+ * Each skill category contains a comma-separated list of abilities.
+ * Abilities can be linked to items using "(via ItemName)" format.
+ *
+ * @param {string} skillsText - The raw skills text from AI response
+ */
+export function parseSkills(skillsText) {
+    debugLog('[RPG Parser] ==================== PARSING SKILLS ====================');
+    debugLog('[RPG Parser] Skills text length:', skillsText?.length + ' chars');
+
+    if (!skillsText || typeof skillsText !== 'string') {
+        debugLog('[RPG Parser] No skills text to parse');
+        return;
+    }
+
+    try {
+        // Initialize data structures if needed
+        if (!extensionSettings.skillsData) {
+            extensionSettings.skillsData = {};
+        }
+        if (!extensionSettings.skillAbilityLinks) {
+            extensionSettings.skillAbilityLinks = {};
+        }
+
+        // Get configured skill categories
+        const configuredCategories = extensionSettings.trackerConfig?.userStats?.skillsSection?.customFields || [];
+        
+        const lines = skillsText.split('\n');
+        const newSkillAbilityLinks = {};
+
+        for (const line of lines) {
+            // Skip header lines and notes
+            if (line.match(/^Skills\s*$/i) || line.match(/^---/) || !line.trim() || line.match(/^\(Note:/i)) {
+                continue;
+            }
+
+            // Parse skill category line: "CategoryName: ability1, ability2 (via Item), ability3"
+            const categoryMatch = line.match(/^(.+?):\s*(.*)$/);
+            if (categoryMatch) {
+                const categoryName = categoryMatch[1].trim();
+                const abilitiesText = categoryMatch[2].trim();
+                
+                // Check if this is a configured category (case-insensitive match)
+                const matchedCategory = configuredCategories.find(c => 
+                    c.toLowerCase() === categoryName.toLowerCase()
+                );
+                
+                if (!matchedCategory) {
+                    debugLog(`[RPG Parser] Skipping unknown skill category: ${categoryName}`);
+                    continue;
+                }
+
+                // Parse abilities (comma-separated)
+                if (!abilitiesText || abilitiesText.toLowerCase() === 'none') {
+                    extensionSettings.skillsData[matchedCategory] = 'None';
+                    debugLog(`[RPG Parser] Skill category ${matchedCategory}: None`);
+                    continue;
+                }
+
+                // Split by comma and parse each ability
+                const abilities = [];
+                const rawAbilities = abilitiesText.split(',').map(a => a.trim()).filter(a => a);
+                
+                for (const rawAbility of rawAbilities) {
+                    // Check for "(ItemName)" pattern - ability granted by item
+                    // Supports both "AbilityName (ItemName)" and legacy "AbilityName (via ItemName)"
+                    const itemMatch = rawAbility.match(/^(.+?)\s*\((?:via\s+)?(.+?)\)$/i);
+                    
+                    if (itemMatch) {
+                        const abilityName = itemMatch[1].trim();
+                        const itemName = itemMatch[2].trim();
+                        abilities.push(abilityName);
+                        
+                        // Store the link
+                        if (extensionSettings.enableItemSkillLinks) {
+                            const linkKey = `${matchedCategory}::${abilityName}`;
+                            newSkillAbilityLinks[linkKey] = itemName;
+                            debugLog(`[RPG Parser] Linked: ${abilityName} <- ${itemName}`);
+                        }
+                    } else {
+                        abilities.push(rawAbility);
+                    }
+                }
+
+                // Store abilities for this category
+                const abilitiesString = abilities.length > 0 ? abilities.join(', ') : 'None';
+                extensionSettings.skillsData[matchedCategory] = abilitiesString;
+                debugLog(`[RPG Parser] Skill category ${matchedCategory}: ${abilitiesString}`);
+            }
+        }
+
+        // Update skill-ability links if item linking is enabled
+        if (extensionSettings.enableItemSkillLinks && Object.keys(newSkillAbilityLinks).length > 0) {
+            // Merge new links with existing ones
+            extensionSettings.skillAbilityLinks = {
+                ...extensionSettings.skillAbilityLinks,
+                ...newSkillAbilityLinks
+            };
+            debugLog('[RPG Parser] Skill-ability links updated:', Object.keys(newSkillAbilityLinks).length + ' new links');
+        }
+
+        saveSettings();
+        debugLog('[RPG Parser] Skills saved successfully');
+        debugLog('[RPG Parser] =======================================================');
+    } catch (error) {
+        console.error('[RPG Companion] Error parsing skills:', error);
+        debugLog('[RPG Parser] ERROR:', error.message);
+    }
+}
+
 /**
  * Helper: Extract code blocks from text
  * @param {string} text - Text containing markdown code blocks