Pakobbix/rpg-companion-sillytavern
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

llm generated image gen prompts

Browse Source
This commit is contained in:
munimunigamer
2025-12-25 21:13:19 -08:00
parent b7e52046bc
commit de11f6f7e2
8 changed files with 256 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
index.js
+24 -11
View File
@@ -36,7 +36,8 @@ import {
setInfoBoxContainer,
setThoughtsContainer,
setInventoryContainer,
setQuestsContainer
setQuestsContainer,
clearSessionAvatarPrompts
} from './src/core/state.js';
import { loadSettings, saveSettings, saveChatData, loadChatData, updateMessageSwipeData } from './src/core/persistence.js';
import { registerAllEvents } from './src/core/events.js';
@@ -406,15 +407,18 @@ async function initUI() {
$('#rpg-toggle-auto-avatars').on('change', function() {
extensionSettings.autoGenerateAvatars = $(this).prop('checked');
saveSettings();
// Show/hide avatar options based on toggle
const $options = $('#rpg-avatar-options');
if (extensionSettings.autoGenerateAvatars) {
$options.slideDown(200);
} else {
$options.slideUp(200);
}
});
$('#rpg-avatar-style-select').on('change', function() {
extensionSettings.avatarGenerationStyle = String($(this).val());
saveSettings();
});
$('#rpg-avatar-custom-prompt').on('input', function() {
extensionSettings.avatarGenerationPrompt = $(this).val().trim();
$('#rpg-avatar-llm-instruction').on('input', function() {
extensionSettings.avatarLLMCustomInstruction = $(this).val().trim();
saveSettings();
});
@@ -509,9 +513,18 @@ async function initUI() {
$('#rpg-toggle-plot-buttons').prop('checked', extensionSettings.enablePlotButtons); $('#rpg-toggle-plot-buttons').prop('checked', extensionSettings.enablePlotButtons); $('#rpg-toggle-plot-buttons').prop('checked', extensionSettings.enablePlotButtons);
$('#rpg-toggle-animations').prop('checked', extensionSettings.enableAnimations);
// Initialize avatar options
$('#rpg-toggle-auto-avatars').prop('checked', extensionSettings.autoGenerateAvatars || false);
$('#rpg-avatar-style-select').val(extensionSettings.avatarGenerationStyle || 'auto');
$('#rpg-avatar-custom-prompt').val(extensionSettings.avatarGenerationPrompt || '');
$('#rpg-avatar-llm-instruction').val(extensionSettings.avatarLLMCustomInstruction || '');
// Initialize avatar options visibility
if (extensionSettings.autoGenerateAvatars) {
$('#rpg-avatar-options').show();
} else {
$('#rpg-avatar-options').hide();
}
$('#rpg-stat-bar-color-low').val(extensionSettings.statBarColorLow);
$('#rpg-stat-bar-color-high').val(extensionSettings.statBarColorHigh);
$('#rpg-theme-select').val(extensionSettings.theme);
@@ -733,7 +746,7 @@ jQuery(async () => {
[event_types.MESSAGE_RECEIVED]: onMessageReceived,
[event_types.GENERATION_STOPPED]: onGenerationEnded,
[event_types.GENERATION_ENDED]: onGenerationEnded,
[event_types.CHAT_CHANGED]: [onCharacterChanged, updatePersonaAvatar, restoreCheckpointOnLoad],
[event_types.CHAT_CHANGED]: [onCharacterChanged, updatePersonaAvatar, restoreCheckpointOnLoad, clearSessionAvatarPrompts],
[event_types.MESSAGE_SWIPED]: onMessageSwiped,
[event_types.USER_MESSAGE_RENDERED]: updatePersonaAvatar,
[event_types.SETTINGS_UPDATED]: updatePersonaAvatar
src/core/state.js
+20 -2
View File
@@ -169,8 +169,7 @@ export let extensionSettings = {
npcAvatars: {}, // Store custom avatar images for NPCs (key: character name, value: base64 data URI)
// Auto avatar generation settings
autoGenerateAvatars: false, // Master toggle for auto-generating avatars
avatarGenerationPrompt: 'portrait, fantasy character, RPG style', // Default prompt template
avatarGenerationStyle: 'auto', // Style preset: auto, fantasy, sci-fi, anime, realistic
avatarLLMCustomInstruction: '', // Custom instruction for LLM prompt generation
};
/**
@@ -193,6 +192,25 @@ export let committedTrackerData = {
characterThoughts: null
};
/**
* Session-only storage for LLM-generated avatar prompts
* Maps character names to their generated prompts
* Resets on new chat (not persisted to extensionSettings)
*/
export let sessionAvatarPrompts = {};
export function setSessionAvatarPrompt(characterName, prompt) {
sessionAvatarPrompts[characterName] = prompt;
}
export function getSessionAvatarPrompt(characterName) {
return sessionAvatarPrompts[characterName] || null;
}
export function clearSessionAvatarPrompts() {
sessionAvatarPrompts = {};
}
/**
* Tracks whether the last action was a swipe (for separate mode)
* Used to determine whether to commit lastGeneratedData to committedTrackerData
src/i18n/en.json
+2 -8
View File
@@ -43,14 +43,8 @@
"template.settingsModal.display.enableDebugModeNote": "Shows parser logs in a mobile-friendly UI panel. Useful for troubleshooting. Look for the red bug button.",
"template.settingsModal.display.autoGenerateAvatars": "Auto-generate Missing Avatars",
"template.settingsModal.display.autoGenerateAvatarsNote": "Automatically generate avatars for characters without custom images using Stable Diffusion",
"template.settingsModal.display.avatarPromptStyle": "Avatar Generation Style:",
"template.settingsModal.display.avatarPromptStyleOptions.auto": "Auto (Fantasy)",
"template.settingsModal.display.avatarPromptStyleOptions.fantasy": "Fantasy",
"template.settingsModal.display.avatarPromptStyleOptions.scifi": "Sci-Fi",
"template.settingsModal.display.avatarPromptStyleOptions.anime": "Anime",
"template.settingsModal.display.avatarPromptStyleOptions.realistic": "Realistic",
"template.settingsModal.display.avatarCustomPrompt": "Custom Avatar Prompt:",
"template.settingsModal.display.avatarCustomPromptNote": "Additional prompt modifiers for avatar generation",
"template.settingsModal.display.avatarLLMInstruction": "LLM Instruction:",
"template.settingsModal.display.avatarLLMInstructionNote": "The LLM will use character cards, tracker data, and chat context to generate detailed prompts",
"template.settingsModal.advancedTitle": "Advanced",
"template.settingsModal.advanced.generationMode": "Generation Mode:",
"template.settingsModal.advanced.generationModeOptions.together": "Together with Main Generation",
src/systems/features/avatarGenerator.js
+16 -15
View File
@@ -4,7 +4,7 @@
*/
import { executeSlashCommandsOnChatInput } from '../../../../../../../scripts/slash-commands.js';
import { extensionSettings } from '../../core/state.js';
import { extensionSettings, sessionAvatarPrompts } from '../../core/state.js';
import { saveSettings } from '../../core/persistence.js';
// Track pending avatar generations to avoid duplicate requests
@@ -16,26 +16,21 @@ const pendingGenerations = new Set();
*/
let onGenerationCompleteCallback = null;
/**
* Style presets for avatar generation prompts
*/
const STYLE_PRESETS = {
'auto': 'portrait, fantasy character, RPG style',
'fantasy': 'portrait, fantasy character, medieval RPG style, detailed face',
'scifi': 'portrait, sci-fi character, futuristic, cyberpunk style, detailed face',
'anime': 'portrait, anime character, manga style, detailed face',
'realistic': 'portrait, realistic character, detailed face, photorealistic'
};
/**
* Builds the generation prompt for a character
* Uses LLM-generated prompt from session storage
* @param {string} characterName - Name of the character
* @returns {string} Full prompt for /sd command
*/
function buildGenerationPrompt(characterName) {
const style = STYLE_PRESETS[extensionSettings.avatarGenerationStyle] || STYLE_PRESETS.auto;
const custom = extensionSettings.avatarGenerationPrompt || '';
return `${style}, ${characterName}, ${custom}`.trim();
const llmPrompt = sessionAvatarPrompts[characterName];
if (llmPrompt) {
console.log(`[RPG Avatar] Using LLM prompt for ${characterName}`);
return llmPrompt;
}
console.warn(`[RPG Avatar] No LLM prompt generated for ${characterName}, skipping generation`);
return null;
}
/**
@@ -84,6 +79,12 @@ export async function generateAvatar(characterName) {
try {
const prompt = buildGenerationPrompt(characterName);
// Skip if no prompt was generated (LLM hasn't generated one yet)
if (!prompt) {
console.log(`[RPG Avatar] No prompt available for ${characterName}, skipping`);
return null;
}
// Execute /sd command with quiet=true
// IMPORTANT: quiet=true must come BEFORE the prompt
// This suppresses chat output and returns the image URL via pipe
src/systems/generation/apiClient.js
+87 -2
View File
@@ -12,10 +12,15 @@ import {
isGenerating,
lastActionWasSwipe,
setIsGenerating,
setLastActionWasSwipe
setLastActionWasSwipe,
sessionAvatarPrompts
} from '../../core/state.js';
import { saveChatData } from '../../core/persistence.js';
import { generateSeparateUpdatePrompt } from './promptBuilder.js';
import {
generateSeparateUpdatePrompt,
generateAvatarPromptGenerationPrompt,
parseAvatarPromptsResponse
} from './promptBuilder.js';
import { parseResponse, parseUserStats } from './parser.js';
import { renderUserStats } from '../rendering/userStats.js';
import { renderInfoBox } from '../rendering/infoBox.js';
@@ -158,6 +163,15 @@ export async function updateRPGData(renderUserStats, renderInfoBox, renderThough
lastGeneratedData.characterThoughts = parsedData.characterThoughts;
}
// Generate avatar prompts if auto-generate is enabled and characters need avatars
if (extensionSettings.autoGenerateAvatars) {
const charactersNeedingPrompts = parseCharactersWithoutAvatars(parsedData.characterThoughts);
if (charactersNeedingPrompts.length > 0) {
console.log('[RPG Companion] Generating LLM avatar prompts for:', charactersNeedingPrompts);
await generateAvatarPrompts(charactersNeedingPrompts);
}
}
// When saveTrackerHistory is enabled, store tracker data on the user's message too
// This allows scrolling through history and seeing trackers at each point
if (extensionSettings.saveTrackerHistory && lastMessage && lastMessage.is_user) {
@@ -253,3 +267,74 @@ export async function updateRPGData(renderUserStats, renderInfoBox, renderThough
setLastActionWasSwipe(false);
}
}
/**
* Parses character thoughts to find characters that need avatar prompts
* @param {string} characterThoughtsData - Raw character thoughts data
* @returns {Array<string>} Array of character names needing prompts
*/
function parseCharactersWithoutAvatars(characterThoughtsData) {
if (!characterThoughtsData) return [];
const lines = characterThoughtsData.split('\n');
const characters = [];
for (const line of lines) {
if (line.trim().startsWith('- ')) {
const name = line.trim().substring(2).trim();
if (name && name.toLowerCase() !== 'unavailable') {
// Skip if already has custom avatar
if (extensionSettings.npcAvatars && extensionSettings.npcAvatars[name]) {
continue;
}
// Skip if already has session prompt
if (sessionAvatarPrompts[name]) {
continue;
}
characters.push(name);
}
}
}
return characters;
}
/**
* Generates LLM-based avatar prompts for specified characters
* Called during batch RPG data refresh when avatar generation is enabled
*
* @param {Array<string>} characterNames - Array of character names needing prompts
* @returns {Promise<Object>} Map of character name to generated prompt
*/
export async function generateAvatarPrompts(characterNames) {
if (!characterNames || characterNames.length === 0) {
return {};
}
try {
console.log('[RPG Avatar] Generating LLM prompts for characters:', characterNames);
const prompt = await generateAvatarPromptGenerationPrompt(characterNames);
// Generate using raw prompt
const response = await generateRaw({
prompt: prompt,
quietToLoud: false
});
if (response) {
const prompts = parseAvatarPromptsResponse(response);
console.log('[RPG Avatar] Generated prompts:', prompts);
// Store in session-only storage
for (const [name, prompt] of Object.entries(prompts)) {
sessionAvatarPrompts[name] = prompt;
}
return prompts;
}
} catch (error) {
console.error('[RPG Avatar] LLM prompt generation failed:', error);
}
return {};
}
src/systems/generation/promptBuilder.js
+82
View File
@@ -597,3 +597,85 @@ export async function generateSeparateUpdatePrompt() {
return messages;
}
/**
* Default custom instruction for avatar prompt generation
*/
const DEFAULT_AVATAR_CUSTOM_INSTRUCTION = `Create a detailed portrait prompt focusing on the character's appearance, clothing, and mood. Include appropriate artistic style keywords.`;
/**
* Generates the prompt for LLM-based avatar prompt generation
* Uses the same context as RPG generation (character cards, tracker data, chat history)
*
* @param {Array<string>} characterNames - Array of character names to generate prompts for
* @returns {Promise<Array<{role: string, content: string}>>} Message array for generateRaw API
*/
export async function generateAvatarPromptGenerationPrompt(characterNames) {
const depth = extensionSettings.updateDepth;
const messages = [];
// Build system message with character context
let systemMessage = `You are an AI assistant specializing in creating detailed image generation prompts for character avatars.\n\n`;
// Add character card information (reusing existing function)
const characterInfo = await getCharacterCardsInfo();
if (characterInfo) {
systemMessage += `Character Information:\n${characterInfo}\n\n`;
}
// Add tracker context if available
if (committedTrackerData.characterThoughts) {
systemMessage += `Current Scene Context:\n${committedTrackerData.characterThoughts}\n\n`;
}
systemMessage += `Recent conversation context:\n<history>`;
messages.push({ role: 'system', content: systemMessage });
// Add chat history
const recentMessages = chat.slice(-depth);
for (const message of recentMessages) {
messages.push({
role: message.is_user ? 'user' : 'assistant',
content: message.mes
});
}
// Build instruction message
let instructionMessage = `</history>\n\n`;
const customInstruction = extensionSettings.avatarLLMCustomInstruction || DEFAULT_AVATAR_CUSTOM_INSTRUCTION;
instructionMessage += `Task: Generate detailed image prompts for the following characters.\n\n`;
instructionMessage += `Instructions: ${customInstruction}\n\n`;
instructionMessage += `Characters:\n`;
characterNames.forEach((name, index) => {
instructionMessage += `${index + 1}. ${name}\n`;
});
instructionMessage += `\nOutput Format (one per line):\n`;
instructionMessage += `CHARACTER_NAME: [detailed prompt]\n\n`;
instructionMessage += `Example:\n`;
instructionMessage += `Gandalf: portrait, elderly wizard with long white beard, wearing grey robes, holding wooden staff, intense blue eyes, wise expression, fantasy art style\n\n`;
instructionMessage += `Provide ONLY the formatted prompts, no other text.`;
messages.push({ role: 'user', content: instructionMessage });
return messages;
}
/**
* Parses LLM response to extract character prompts
* @param {string} response - Raw LLM response
* @returns {Object} Map of character name to prompt
*/
export function parseAvatarPromptsResponse(response) {
const prompts = {};
const lines = response.split('\n');
for (const line of lines) {
const trimmed = line.trim();
const match = trimmed.match(/^([^:]+):\s*(.+)$/);
if (match) {
prompts[match[1].trim()] = match[2].trim();
}
}
return prompts;
}
style.css
+14
View File
@@ -6787,3 +6787,17 @@ body:has(.rpg-panel.rpg-position-left) #sheld {
font-size: 24px;
border-radius: 8px;
}
/* Textarea style for LLM instruction input */
.rpg-textarea {
width: 100%;
min-height: 80px;
padding: 10px;
border-radius: 4px;
border: 1px solid var(--rpg-border);
background: var(--SmartThemeBlurTintColor);
color: var(--SmartThemeBodyColor);
font-family: inherit;
resize: vertical;
box-sizing: border-box;
}
template.html
+11 -19
View File
@@ -238,25 +238,17 @@
Automatically generate avatars for characters without custom images using Stable Diffusion
</small>
<div class="rpg-setting-row" style="margin-top: 12px;">
<label for="rpg-avatar-style-select" data-i18n-key="template.settingsModal.display.avatarPromptStyle">Avatar Generation Style:</label>
<select id="rpg-avatar-style-select" class="rpg-select">
<option value="auto" data-i18n-key="template.settingsModal.display.avatarPromptStyleOptions.auto">Auto (Fantasy)</option>
<option value="fantasy" data-i18n-key="template.settingsModal.display.avatarPromptStyleOptions.fantasy">Fantasy</option>
<option value="scifi" data-i18n-key="template.settingsModal.display.avatarPromptStyleOptions.scifi">Sci-Fi</option>
<option value="anime" data-i18n-key="template.settingsModal.display.avatarPromptStyleOptions.anime">Anime</option>
<option value="realistic" data-i18n-key="template.settingsModal.display.avatarPromptStyleOptions.realistic">Realistic</option>
</select>
</div>
<div class="rpg-setting-row" style="margin-top: 12px;">
<label for="rpg-avatar-custom-prompt" style="display: block; margin-bottom: 8px;" data-i18n-key="template.settingsModal.display.avatarCustomPrompt">
Custom Avatar Prompt:
</label>
<input type="text" id="rpg-avatar-custom-prompt" class="rpg-input" placeholder="portrait, fantasy character, RPG style" />
<small style="display: block; margin-top: 4px; color: #888; font-size: 11px;" data-i18n-key="template.settingsModal.display.avatarCustomPromptNote">
Additional prompt modifiers for avatar generation
</small>
<!-- Avatar options container - conditionally visible -->
<div id="rpg-avatar-options" style="margin-left: 24px; margin-top: 12px; display: none;">
<div class="rpg-setting-row" style="margin-top: 12px;">
<label for="rpg-avatar-llm-instruction" style="display: block; margin-bottom: 8px;" data-i18n-key="template.settingsModal.display.avatarLLMInstruction">
LLM Instruction:
</label>
<textarea id="rpg-avatar-llm-instruction" class="rpg-textarea" rows="3" placeholder="Create a detailed portrait prompt focusing on the character's appearance, clothing, and mood..."></textarea>
<small style="display: block; margin-top: 4px; color: #888; font-size: 11px;" data-i18n-key="template.settingsModal.display.avatarLLMInstructionNote">
The LLM will use character cards, tracker data, and chat context to generate detailed prompts
</small>
</div>
</div>
</div>