// AiNpcControl.cs // A talking, listening, walking AI NPC. It wears one of YOUR (the scene owner's) saved outfits, follows players, // and holds a spoken conversation powered by your own Claude or ChatGPT API key. // // How the pieces fit together (each is usable on its own; this script just wires them up): // * AiNpc - an avatar clone (from a named outfit) the SERVER walks around: WalkTo / Follow / StopMoving, // plus Say (speaks aloud to everyone nearby, quieter with distance) and Listen (one player's mic). // * AiBrain - the conversation: Provider + [Secret] ApiKey + Instructions + Knowledge, Ask(text, reply => ...). // * CreatePanel- the little worldspace menu (Follow me / Interact) that pops up when you click the NPC. // // The API key is a [Secret] field: it is stored only on the server, never downloaded by players, and only you // (the scene owner) can see or change it in the Inspector. // // Setup: drop this script on an empty object where the NPC should appear. Set OutfitOwner to your username and // OutfitName to one of your saved outfits. Pick a Provider and paste the matching API key. Describe how the NPC // should behave in Instructions, and give it facts in Knowledge. Walk up to it and click it. using System.Collections; using System.Collections.Generic; using IHXR.Simulation; public class AiNpcControl : ScriptBase { public enum AiProvider { Anthropic, OpenAI } public enum SpeechEngine { Browser, Local } public enum NpcVoice { Default, Female, Male, GoogleUSEnglish, GoogleUKFemale, GoogleUKMale } [Header("Who the NPC is")] [Tooltip("The display name shown when it talks in chat.")] public string NpcName = "Guide"; [Tooltip("Whose saved outfit the NPC wears. Your username (the scene owner).")] public string OutfitOwner = ""; [Tooltip("The name of one of that person's outfits (Avatar Editor). Blank = the outfit they currently wear.")] public string OutfitName = ""; public Vector3 SpawnOffset = new Vector3(0f, 0f, 0f); public Vector3 SpawnFacing = Vector3.zero; [Header("Brain (your own API key)")] public AiProvider Provider = AiProvider.Anthropic; [Secret] [Tooltip("Your Anthropic or OpenAI API key. Stored on the server only; players never receive it.")] public string ApiKey = ""; [Tooltip("Model id. Light + cheap by default (good for short NPC chat). Anthropic: claude-haiku-5-5 / claude-sonnet-5-5. If you switch Provider to OpenAI it uses gpt-4o-mini automatically.")] public string Model = "claude-haiku-5-5"; [Multiline(4)] [Tooltip("How the NPC should behave, talk and act — its personality and rules.")] public string Instructions = "You are a friendly, concise guide in a virtual world. Keep replies to a sentence or two, suitable for speaking aloud."; [Multiline(4)] [Tooltip("Facts the NPC may use when answering (its knowledge bank).")] public string Knowledge = ""; [Range(50, 800)] public int MaxReplyTokens = 300; [Header("Voice")] [Tooltip("Which voice the NPC speaks with. Default = the player's system voice. Female/Male pick a matching voice on each player's device. The Google ones use those voices when present, else fall back to the default.")] public NpcVoice Voice = NpcVoice.Female; [Range(0.5f, 2f)] public float VoiceRate = 1f; [Range(0f, 2f)] public float VoicePitch = 1f; [Tooltip("How the player's speech is turned into text when they Interact. Browser = the browser's built-in recognition (fast, but streams audio to the browser's cloud). Local = a small Whisper model that runs fully in the player's browser (private, offline; a ~75MB one-time download).")] public SpeechEngine VoiceToText = SpeechEngine.Browser; [Header("Movement")] [Range(1f, 8f)] public float WalkSpeed = 3.5f; [Tooltip("How close the NPC stops when following a player.")] public float FollowDistance = 2f; [Header("Menu")] [Tooltip("How close you must be to click the NPC.")] public float ClickDistance = 14f; public float MenuHeight = 2.55f; [Range(0.4f, 3f)] public float MenuWidth = 0.9f; [Tooltip("Seconds the pop-up menu stays up before it fades if you don't pick anything.")] public float MenuIdleSeconds = 6f; AiNpc npc; AiBrain brain; ScriptPanel menu; int menuToken; bool interacting; string interactingWith = ""; string followingWho = ""; int interactToken; [Tooltip("How far (metres) the NPC looks for a seat. Any sitpoint in the scene within this range is used automatically — no need to slot any in.")] public float SeatSearchRange = 20f; [Tooltip("When a player asks what something looks like, the NPC snapshots their view and the model describes it. Max such requests per hour (0 = off).")] public int VisionRepliesPerHour = 6; bool seated; Sitpoint currentSeat; readonly List visionTimes = new List(); public override void Start() { string owner = string.IsNullOrWhiteSpace(OutfitOwner) ? "" : OutfitOwner.Trim(); if (owner == "") { Log("[AiNpc] Set OutfitOwner to your username (the scene owner) so the NPC knows whose outfit to wear."); return; } Vector3 spawnPos = transform.position + SpawnOffset; AvatarClone clone = SpawnAvatarClone(owner, spawnPos, SpawnFacing, Vector3.one, OutfitName); npc = new AiNpc(clone, spawnPos) { WalkSpeed = WalkSpeed }; // Click the NPC itself to open the menu. MakeInteractable(clone, OnNpcClicked, true, null, true, ClickDistance, "Talk to " + NpcName); brain = new AiBrain { Provider = Provider.ToString(), ApiKey = ApiKey, Model = Model, Instructions = Instructions, Knowledge = Knowledge, MaxTokens = MaxReplyTokens, }; // The worldspace pop-up menu (hidden until the NPC is clicked). menu = CreatePanel("ai-npc-menu", 300); menu.SetTitle(NpcName).SetLayout(1, 8, 14); menu.AddButton("Follow me", OnFollowClicked); menu.AddButton("Stop following", OnStopFollowClicked); menu.AddButton("Interact", OnInteractClicked); menu.AddButton("Sit / Stand", OnSitClicked); // Stand the menu above the NPC so it rides along wherever the NPC walks (not a fixed spot). menu.PlaceOnObject(npc.Clone, new Vector3(0f, MenuHeight, 0f), MenuWidth, true); if (!brain.HasKey) Log("[AiNpc] No API key set — the NPC can walk and follow, but can't talk until you add your key in the Inspector."); Log("[AiNpc] '" + NpcName + "' ready (outfit of " + owner + (OutfitName == "" ? "" : " / '" + OutfitName + "'") + ")."); } void OnNpcClicked(InteractionInfo info) { if (npc == null || npc.IsNull || info == null) return; ShowMenuTo(info.Username); } void ShowMenuTo(string username) { if (string.IsNullOrWhiteSpace(username) || menu == null) return; menu.Show(username); // Idle fade: after MenuIdleSeconds, hide it unless another action superseded this one. int token = ++menuToken; StartCoroutine(HideMenuAfterIdle(token, username)); } IEnumerator HideMenuAfterIdle(int token, string username) { yield return new WaitForSeconds(MenuIdleSeconds); if (token == menuToken && menu != null) menu.Hide(username); } void OnFollowClicked(PanelEvent e) { menuToken++; // cancels the pending idle-hide menu.Hide(e.Username); if (npc == null || npc.IsNull) return; if (seated) StandUp(); followingWho = e.Username; npc.Follow(e.Username, FollowDistance); SendScriptChat(NpcName + " is now following " + e.Username + "."); } void OnStopFollowClicked(PanelEvent e) { menuToken++; menu.Hide(e.Username); if (npc == null || npc.IsNull) return; if (!string.IsNullOrEmpty(followingWho)) SendScriptChat(NpcName + " stopped following " + followingWho + "."); followingWho = ""; npc.StopMoving(); } void OnSitClicked(PanelEvent e) { menuToken++; menu.Hide(e.Username); if (npc == null || npc.IsNull) return; if (seated) { StandUp(); SendScriptChat(NpcName + " stood up."); return; } GoSitNearest(e.Username); } // Closest sitpoint in the scene to the NPC, within SeatSearchRange (found automatically — no slotting required). Sitpoint NearestSeat() { if (npc == null || npc.IsNull) return null; return NearestSitpoint(npc.Position, SeatSearchRange); } // Walk to the nearest seat and sit in it. Movement runs on the server, so every player sees the same walk + sit. void GoSitNearest(string username) { Sitpoint seat = NearestSeat(); if (seat == null) { SendScriptChat(NpcName + " can't find a seat nearby."); return; } followingWho = ""; currentSeat = seat; SendScriptChat(NpcName + " is heading to a seat."); // Stop ~1.4m out so the NPC halts at the edge of the seat/booth instead of grinding into it; Sit snaps it onto the seat. npc.WalkTo(seat.transform.position, 12f, 1.4f, ok => { if (npc == null || npc.IsNull || !ok) return; npc.Sit(seat); seated = true; }); } void StandUp() { if (npc == null || npc.IsNull) return; if (seated) npc.Stand(); seated = false; currentSeat = null; } void OnInteractClicked(PanelEvent e) { menuToken++; menu.Hide(e.Username); if (npc == null || npc.IsNull) return; if (interacting) { // Same person clicking again (after a rejoin, or to retry): let them reclaim the conversation instead // of being told the NPC is "busy" with themselves. Only a DIFFERENT player is asked to wait. if (string.Equals(e.Username, interactingWith, System.StringComparison.OrdinalIgnoreCase)) { npc.StopListening(interactingWith); EndInteraction(); } else { SendScriptChat(NpcName + " is busy talking with " + interactingWith + " — one at a time!"); return; } } if (!brain.HasKey) { SendScriptChat(NpcName + " has no API key set, so it can't chat yet."); return; } interacting = true; interactingWith = e.Username; int convToken = ++interactToken; StartCoroutine(InteractionWatchdog(convToken)); // Stop and face the player, then listen to their microphone. npc.StopMoving(); followingWho = ""; UserInfo user = GetListener(e.Username); if (user != null) npc.FaceTowards(user.Position); npc.Listen(e.Username, heard => { string said = (heard ?? "").Trim(); if (said == "") { Speak("Sorry, I didn't catch that. Click Interact and try again."); EndInteraction(); return; } // Everyone sees what the player said (and hears them over normal voice chat). SendScriptChat(e.Username + ": " + said); // Let the player ask the NPC, by voice, to sit down or stand up. string lower = said.ToLower(); if (!seated && (lower.Contains("sit") || lower.Contains("have a seat") || lower.Contains("take a seat"))) { Speak("Sure, let me grab a seat."); GoSitNearest(e.Username); EndInteraction(); return; } if (seated && (lower.Contains("stand") || lower.Contains("get up"))) { Speak("Okay, standing up."); StandUp(); EndInteraction(); return; } // Vision: if the player asks about what they're looking at, snapshot their view and let the model describe // it. Rate-limited (VisionRepliesPerHour); over the limit it just falls through to a normal spoken reply. if (IsVisionQuestion(lower) && VisionAllowed()) { visionTimes.Add(DateTime.UtcNow); Speak("Let me take a look..."); npc.LookAndAsk(e.Username, said, brain, reply => { string answer = (reply ?? "").Trim(); if (answer == "") answer = "Hmm, I can't quite make that out from here."; SendScriptChat(NpcName + ": " + answer); Speak(answer); EndInteraction(); }); return; } brain.Ask(said, e.Username, reply => { string answer = (reply ?? "").Trim(); if (answer == "") answer = "Hmm, I'm not sure how to answer that."; SendScriptChat(NpcName + ": " + answer); Speak(answer); EndInteraction(); }); }, VoiceToText == SpeechEngine.Local ? "local" : "browser"); } // Does the player's line ask about what they can see? (A light heuristic; tune the phrases for your world.) bool IsVisionQuestion(string lower) { return lower.Contains("look like") || lower.Contains("what do you see") || lower.Contains("what do u see") || lower.Contains("look at") || lower.Contains("take a look") || lower.Contains("what is this") || lower.Contains("what's this") || lower.Contains("whats this") || lower.Contains("describe what") || lower.Contains("what am i looking") || lower.Contains("can you see") || lower.Contains("what color"); } // True while under the per-hour vision cap; also prunes entries older than an hour. bool VisionAllowed() { DateTime now = DateTime.UtcNow; visionTimes.RemoveAll(t => (now - t).TotalHours >= 1.0); return visionTimes.Count < Math.Max(0, VisionRepliesPerHour); } void Speak(string text) { if (npc != null && !npc.IsNull) npc.Say(text, VoiceRate, VoicePitch, VoiceName()); } string VoiceName() { switch (Voice) { case NpcVoice.Female: return "female"; case NpcVoice.Male: return "male"; case NpcVoice.GoogleUSEnglish: return "Google US English"; case NpcVoice.GoogleUKFemale: return "Google UK English Female"; case NpcVoice.GoogleUKMale: return "Google UK English Male"; default: return ""; } } void EndInteraction() { interacting = false; interactingWith = ""; interactToken++; // cancels any pending watchdog } // Safety net: if a conversation never completes (e.g. speech-to-text returns nothing and no callback fires), // release the one-at-a-time lock after a while so the NPC never gets stuck "busy". IEnumerator InteractionWatchdog(int token) { yield return new WaitForSeconds(90f); if (token == interactToken && interacting) { if (!string.IsNullOrEmpty(interactingWith)) npc?.StopListening(interactingWith); EndInteraction(); } } UserInfo GetListener(string username) { foreach (UserInfo u in GetUsers()) if (u != null && string.Equals(u.Username, username, System.StringComparison.OrdinalIgnoreCase)) return u; return null; } public override void OnUserJoin(UserInfo user) { // If the player who held the conversation lock left and has now rejoined (their old session is gone), // clear the stale lock so they — and everyone else — can interact again. if (user != null && interacting && string.Equals(user.Username, interactingWith, System.StringComparison.OrdinalIgnoreCase)) { npc?.StopListening(interactingWith); EndInteraction(); } } public override void OnUserPart(UserInfo user) { if (user == null) return; if (string.Equals(user.Username, followingWho, System.StringComparison.OrdinalIgnoreCase)) { followingWho = ""; npc?.StopMoving(); } if (interacting && string.Equals(user.Username, interactingWith, System.StringComparison.OrdinalIgnoreCase)) { npc?.StopListening(user.Username); EndInteraction(); } } public override void OnDestroy() { npc?.Destroy(); } }