diff --git a/AIDevGallery/App.xaml b/AIDevGallery/App.xaml index 2f8f3256..8475788b 100644 --- a/AIDevGallery/App.xaml +++ b/AIDevGallery/App.xaml @@ -31,12 +31,44 @@ ms-appx:///Assets/ModelIcons/GitHub.light.svg + + #ee9bbf + + + + + + ms-appx:///Assets/ModelIcons/GitHub.dark.svg + + #ee9bbf + + + + + + ms-appx:///Assets/ModelIcons/GitHub.dark.svg + + #48B1E9 + diff --git a/AIDevGallery/Assets/InteriorDesign.png b/AIDevGallery/Assets/InteriorDesign.png new file mode 100644 index 00000000..0556e2be Binary files /dev/null and b/AIDevGallery/Assets/InteriorDesign.png differ diff --git a/AIDevGallery/Assets/ShakshukaRecipe.png b/AIDevGallery/Assets/ShakshukaRecipe.png new file mode 100644 index 00000000..72357462 Binary files /dev/null and b/AIDevGallery/Assets/ShakshukaRecipe.png differ diff --git a/AIDevGallery/Assets/TofuBowlRecipe.png b/AIDevGallery/Assets/TofuBowlRecipe.png new file mode 100644 index 00000000..e800d900 Binary files /dev/null and b/AIDevGallery/Assets/TofuBowlRecipe.png differ diff --git a/AIDevGallery/Controls/SampleContainer.xaml.cs b/AIDevGallery/Controls/SampleContainer.xaml.cs index ba455504..30196b9d 100644 --- a/AIDevGallery/Controls/SampleContainer.xaml.cs +++ b/AIDevGallery/Controls/SampleContainer.xaml.cs @@ -231,7 +231,8 @@ public async Task LoadSampleAsync(Sample? sample, List? models, Wi _wcrApi = apiType; VisualStateManager.GoToState(this, "WcrModelNeedsDownload", true); - if (!await modelDownloader.SetDownloadOperation(apiType, sample.Id, WcrApiHelpers.EnsureReadyFuncs[apiType]).WaitAsync(token)) + if (!ModelDetailsHelper.IsACIApi(wcrApi) && + !await modelDownloader.SetDownloadOperation(apiType, sample.Id, WcrApiHelpers.EnsureReadyFuncs[apiType]).WaitAsync(token)) { return; } diff --git a/AIDevGallery/Helpers/ModelDetailsHelper.cs b/AIDevGallery/Helpers/ModelDetailsHelper.cs index b1d06dfc..8f9bed32 100644 --- a/AIDevGallery/Helpers/ModelDetailsHelper.cs +++ b/AIDevGallery/Helpers/ModelDetailsHelper.cs @@ -33,12 +33,24 @@ public static bool EqualOrParent(ModelType modelType, ModelType searchModelType) public static ModelDetails GetModelDetailsFromApiDefinition(ModelType modelType, ApiDefinition apiDefinition) { + List hardwareAccelerators; + + // ACI is a subset of WCRAPIs but without the same set of hardware restrictions. Adding exception here. + if (apiDefinition.Category == "App Content Search") + { + hardwareAccelerators = [HardwareAccelerator.WCRAPI, HardwareAccelerator.ACI]; + } + else + { + hardwareAccelerators = [HardwareAccelerator.WCRAPI]; + } + return new ModelDetails { Id = apiDefinition.Id, Icon = apiDefinition.Icon, Name = apiDefinition.Name, - HardwareAccelerators = [HardwareAccelerator.WCRAPI], + HardwareAccelerators = hardwareAccelerators, IsUserAdded = false, SupportedOnQualcomm = true, ReadmeUrl = apiDefinition.ReadmeUrl, @@ -183,6 +195,11 @@ public static bool IsHttpApi(this ExpandedModelDetails modelDetails) return ExternalModelHelper.HardwareAccelerators.Contains(modelDetails.HardwareAccelerator); } + public static bool IsACIApi(this ModelDetails modelDetails) + { + return modelDetails.HardwareAccelerators.Contains(HardwareAccelerator.ACI); + } + public static bool IsLanguageModel(this ModelDetails modelDetails) { return modelDetails.HardwareAccelerators.Contains(HardwareAccelerator.OLLAMA) || diff --git a/AIDevGallery/MainWindow.xaml b/AIDevGallery/MainWindow.xaml index 4fb4c95e..932203bd 100644 --- a/AIDevGallery/MainWindow.xaml +++ b/AIDevGallery/MainWindow.xaml @@ -153,10 +153,17 @@ VerticalAlignment="Center" ItemTemplateSelector="{StaticResource SearchResultTemplateSelector}" PlaceholderText="Search samples, models & APIs.." - QueryIcon="Find" QuerySubmitted="SearchBox_QuerySubmitted" TextChanged="SearchBox_TextChanged" - UpdateTextOnSelect="False" /> + UpdateTextOnSelect="False"> + + + + modelOrApiPicker; public MainWindow(object? obj = null) @@ -43,6 +51,61 @@ public MainWindow(object? obj = null) Close(); } }; + + if (App.AppData.IsAppContentSearchEnabled) + { + Task.Run(async () => + { + // Load AppContentSearch + await LoadAppSearchIndex(); + }); + } + + App.AppData.PropertyChanged += AppData_PropertyChanged; + } + + private async Task LoadAppSearchIndex() + { + var result = AppContentIndexer.GetOrCreateIndex("AIDevGallerySearchIndex"); + + if (!result.Succeeded) + { + throw new InvalidOperationException($"Failed to open index. Status = '{result.Status}', Error = '{result.ExtendedError}'"); + } + + _indexer = result.Indexer; + await _indexer.WaitForIndexCapabilitiesAsync(); + + // If result.Succeeded is true, result.Status will either be CreatedNew or OpenedExisting + if (result.Status == GetOrCreateIndexStatus.CreatedNew || !App.AppData.IsAppContentIndexCompleted) + { + Debug.WriteLine("Created a new index"); + IndexContentsWithAppContentSearch(); + } + else if (result.Status == GetOrCreateIndexStatus.OpenedExisting) + { + Debug.WriteLine("Opened an existing index"); + SetSearchBoxIndexingCompleted(); + } + } + + private void AppData_PropertyChanged(object? sender, PropertyChangedEventArgs e) + { + if (e.PropertyName == nameof(AppData.IsAppContentSearchEnabled)) + { + if (App.AppData.IsAppContentSearchEnabled) + { + Task.Run(async () => + { + // Load AppContentSearch + await LoadAppSearchIndex(); + }); + } + else + { + SetSearchBoxACSDisabled(); + } + } } public void NavigateToPage(object? obj) @@ -220,17 +283,64 @@ private void ManageModelsClicked(object sender, RoutedEventArgs e) NavFrame.Navigate(typeof(SettingsPage), "ModelManagement"); } - private void SearchBox_TextChanged(AutoSuggestBox sender, AutoSuggestBoxTextChangedEventArgs args) + private async void SearchBox_TextChanged(AutoSuggestBox sender, AutoSuggestBoxTextChangedEventArgs args) { if (args.Reason == AutoSuggestionBoxTextChangeReason.UserInput && !string.IsNullOrWhiteSpace(SearchBox.Text)) { - var filteredSearchResults = App.SearchIndex.Where(sr => sr.Label.Contains(sender.Text, StringComparison.OrdinalIgnoreCase)).ToList(); - var orderedResults = filteredSearchResults.OrderByDescending(i => i.Label.StartsWith(sender.Text, StringComparison.CurrentCultureIgnoreCase)).ThenBy(i => i.Label).ToList(); - SearchBox.ItemsSource = orderedResults; + // Cancel previous search if running + _searchCts?.Cancel(); + _searchCts = new CancellationTokenSource(); + var token = _searchCts.Token; + var searchText = sender.Text; + List orderedResults = new(); - var resultCount = orderedResults.Count; - string announcement = $"Searching for '{sender.Text}', {resultCount} search result{(resultCount == 1 ? string.Empty : 's')} found"; - NarratorHelper.Announce(SearchBox, announcement, "searchSuggestionsActivityId"); + try + { + if (_indexer != null && App.AppData.IsAppContentSearchEnabled) + { + // Use AppContentIndexer to search + var query = _indexer.CreateQuery(searchText); + IReadOnlyList? matches = await Task.Run(() => query.GetNextTextMatches(5), token); + + if (!token.IsCancellationRequested && matches != null && matches.Count > 0) + { + foreach (var match in matches) + { + if (token.IsCancellationRequested) + { + break; + } + + var sr = App.SearchIndex.FirstOrDefault(s => s.Label == match.ContentId); + if (sr != null) + { + orderedResults.Add(sr); + } + } + } + } + else + { + // Fallback to in-memory search + var filteredSearchResults = App.SearchIndex.Where(sr => sr.Label.Contains(searchText, StringComparison.OrdinalIgnoreCase)).ToList(); + orderedResults = filteredSearchResults + .OrderByDescending(i => i.Label.StartsWith(searchText, StringComparison.CurrentCultureIgnoreCase)) + .ThenBy(i => i.Label) + .ToList(); + } + + if (!token.IsCancellationRequested) + { + SearchBox.ItemsSource = orderedResults; + var resultCount = orderedResults.Count; + string announcement = $"Searching for '{searchText}', {resultCount} search result{(resultCount == 1 ? string.Empty : "s")} found"; + NarratorHelper.Announce(SearchBox, announcement, "searchSuggestionsActivityId"); + } + } + catch (OperationCanceledException) + { + // Search was cancelled, do nothing + } } } @@ -290,4 +400,56 @@ private void NavFrame_Navigated(object sender, Microsoft.UI.Xaml.Navigation.Navi titleBarIcon.Margin = new Thickness(16, 0, 0, 0); } } + + private void SetSearchBoxIndexingCompleted() + { + DispatcherQueue.TryEnqueue(() => + { + SearchBoxQueryIcon.Foreground = Application.Current.Resources["AIAccentGradientBrush"] as Brush; + SearchBoxQueryIcon.Glyph = "\uED37"; + }); + } + + private void SetSearchBoxACSDisabled() + { + DispatcherQueue.TryEnqueue(() => + { + SearchBoxQueryIcon.Foreground = Application.Current.Resources["TextFillColorPrimaryBrush"] as Brush; + SearchBoxQueryIcon.Glyph = "\uE721"; + }); + } + + private async void IndexContentsWithAppContentSearch() + { + if (_indexer == null || App.SearchIndex == null) + { + SetSearchBoxACSDisabled(); + return; + } + + await Task.Run(() => + { + foreach (var item in App.SearchIndex) + { + string id = item.Label; + string value = $"{item.Label}\n{item.Description}"; + IndexableAppContent textContent = AppManagedIndexableAppContent.CreateFromString(id, value); + _indexer.AddOrUpdate(textContent); + } + }); + + await _indexer.WaitForIndexingIdleAsync(50000); + SetSearchBoxIndexingCompleted(); + + // Adding a check here since if the user closes in the middle of the indexing loop, we will never fully finish indexing. + // The next app launch will open the existing index and consider everything done. + App.AppData.IsAppContentIndexCompleted = true; + await App.AppData.SaveAsync(); + } + + public static void IndexAppSearchIndexStatic() + { + var mainWindow = (MainWindow)App.MainWindow; + mainWindow?.IndexContentsWithAppContentSearch(); + } } \ No newline at end of file diff --git a/AIDevGallery/Models/ModelCompatibility.cs b/AIDevGallery/Models/ModelCompatibility.cs index b0f36e30..2676212f 100644 --- a/AIDevGallery/Models/ModelCompatibility.cs +++ b/AIDevGallery/Models/ModelCompatibility.cs @@ -26,7 +26,11 @@ public static ModelCompatibility GetModelCompatibility(ModelDetails modelDetails string description = string.Empty; ModelCompatibilityState compatibility; - if (modelDetails.IsHttpApi()) + if (modelDetails.IsACIApi()) + { + compatibility = ModelCompatibilityState.Compatible; + } + else if (modelDetails.IsHttpApi()) { compatibility = ModelCompatibilityState.Compatible; } diff --git a/AIDevGallery/Models/Samples.cs b/AIDevGallery/Models/Samples.cs index 4c3a3002..3163d476 100644 --- a/AIDevGallery/Models/Samples.cs +++ b/AIDevGallery/Models/Samples.cs @@ -169,6 +169,7 @@ internal class Scenario [JsonConverter(typeof(JsonStringEnumConverter))] internal enum HardwareAccelerator { + ACI, CPU, DML, QNN, diff --git a/AIDevGallery/Pages/APIs/APISelectionPage.xaml b/AIDevGallery/Pages/APIs/APISelectionPage.xaml index b38853d1..89e3b8ad 100644 --- a/AIDevGallery/Pages/APIs/APISelectionPage.xaml +++ b/AIDevGallery/Pages/APIs/APISelectionPage.xaml @@ -16,7 +16,7 @@ IsPaneToggleButtonVisible="True" IsPaneVisible="True" IsSettingsVisible="False" - OpenPaneLength="224" + OpenPaneLength="276" PaneDisplayMode="Auto" SelectionChanged="NavView_SelectionChanged"> diff --git a/AIDevGallery/Pages/APIs/APISelectionPage.xaml.cs b/AIDevGallery/Pages/APIs/APISelectionPage.xaml.cs index 0430295d..8803a331 100644 --- a/AIDevGallery/Pages/APIs/APISelectionPage.xaml.cs +++ b/AIDevGallery/Pages/APIs/APISelectionPage.xaml.cs @@ -113,6 +113,6 @@ public void SetSelectedApiInMenu(ModelType selectedType) public void ShowHideNavPane() { - NavView.OpenPaneLength = NavView.OpenPaneLength == 0 ? 224 : 0; + NavView.OpenPaneLength = NavView.OpenPaneLength == 0 ? 276 : 0; } } \ No newline at end of file diff --git a/AIDevGallery/Pages/Scenarios/ScenarioPage.xaml.cs b/AIDevGallery/Pages/Scenarios/ScenarioPage.xaml.cs index acbf21d4..b623da21 100644 --- a/AIDevGallery/Pages/Scenarios/ScenarioPage.xaml.cs +++ b/AIDevGallery/Pages/Scenarios/ScenarioPage.xaml.cs @@ -315,7 +315,7 @@ private void LoadSample(Sample? sampleToLoad) // TODO: don't load sample if model is not cached, but still let code to be seen // this would probably be handled in the SampleContainer - _ = SampleContainer.LoadSampleAsync(sample, [.. modelDetails], App.AppData.WinMLSampleOptions); + _ = SampleContainer.LoadSampleAsync(sample, modelDetails.Where(m => m != null).Cast().ToList(), App.AppData.WinMLSampleOptions); _ = App.AppData.AddMru( new MostRecentlyUsedItem() { diff --git a/AIDevGallery/Pages/Scenarios/ScenarioSelectionPage.xaml.cs b/AIDevGallery/Pages/Scenarios/ScenarioSelectionPage.xaml.cs index 37ffd20b..d158a689 100644 --- a/AIDevGallery/Pages/Scenarios/ScenarioSelectionPage.xaml.cs +++ b/AIDevGallery/Pages/Scenarios/ScenarioSelectionPage.xaml.cs @@ -100,7 +100,7 @@ public void HandleNavigation(object? obj) public void ShowHideNavPane() { - NavView.OpenPaneLength = NavView.OpenPaneLength == 0 ? 248 : 0; + NavView.OpenPaneLength = NavView.OpenPaneLength == 0 ? 276 : 0; } private void SetUpScenarios(string? filter = null) diff --git a/AIDevGallery/Pages/SettingsPage.xaml b/AIDevGallery/Pages/SettingsPage.xaml index ddab5f49..ac9956e4 100644 --- a/AIDevGallery/Pages/SettingsPage.xaml +++ b/AIDevGallery/Pages/SettingsPage.xaml @@ -105,6 +105,22 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/AIDevGallery/Samples/WCRAPIs/KnowledgeRetrieval.xaml.cs b/AIDevGallery/Samples/WCRAPIs/KnowledgeRetrieval.xaml.cs new file mode 100644 index 00000000..ad93606a --- /dev/null +++ b/AIDevGallery/Samples/WCRAPIs/KnowledgeRetrieval.xaml.cs @@ -0,0 +1,712 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using AIDevGallery.Models; +using AIDevGallery.Samples.Attributes; +using AIDevGallery.Samples.SharedCode; +using Microsoft.Extensions.AI; +using Microsoft.UI.Xaml; +using Microsoft.UI.Xaml.Controls; +using Microsoft.UI.Xaml.Input; +using Microsoft.UI.Xaml.Media; +using Microsoft.UI.Xaml.Navigation; +using Microsoft.Windows.AI.Search.Experimental.AppContentIndex; +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Windows.Graphics.Imaging; +using Windows.Storage; + +namespace AIDevGallery.Samples.WCRAPIs; + +[GallerySample( + Name = "Knowledge Retrieval (RAG)", + Model1Types = [ModelType.KnowledgeRetrieval, ModelType.PhiSilica], + Scenario = ScenarioType.TextRetrievalAugmentedGeneration, + Id = "6a526fdd-359f-4eac-9aa6-f01db11ae542", + SharedCode = [ + SharedCodeEnum.DataItems, + SharedCodeEnum.Message, + SharedCodeEnum.ChatTemplateSelector + ], + NugetPackageReferences = [ + "CommunityToolkit.Mvvm", + "Microsoft.Extensions.AI", + "Microsoft.WindowsAppSDK" + ], + Icon = "\uEE6F")] + +internal sealed partial class KnowledgeRetrieval : BaseSamplePage +{ + private ObservableCollection TextDataItems { get; } = new(); + public ObservableCollection Messages { get; } = []; + + private IChatClient? _model; + private ScrollViewer? _scrollViewer; + private bool _isImeActive = true; + + // Markers for the assistant's think area (displayed in a dedicated UI region). + private static readonly string[] ThinkTagOpens = new[] { "", "", "" }; + private static readonly string[] ThinkTagCloses = new[] { "", "", "" }; + private static readonly int MaxOpenThinkMarkerLength = ThinkTagOpens.Max(s => s.Length); + + // This is some text data that we want to add to the index: + private Dictionary simpleTextData = new Dictionary + { + { "item1", "Preparing a hearty vegetable stew begins with chopping fresh carrots, onions, and celery. Sauté them in olive oil until fragrant, then add diced tomatoes, herbs, and vegetable broth. Simmer gently for an hour, allowing flavors to meld into a comforting dish perfect for cold evenings." }, + { "item2", "Modern exhibition design combines narrative flow with spatial strategy. Lighting emphasizes focal objects while circulation paths avoid bottlenecks. Materials complement artifacts without visual competition. Interactive elements invite engagement but remain intuitive. Environmental controls protect sensitive works. Success balances scholarship, aesthetics, and visitor experience through thoughtful, cohesive design choices." }, + { "item3", "Domestic cats communicate through posture, tail flicks, and vocalizations. Play mimics hunting behaviors like stalking and pouncing, supporting agility and mental stimulation. Scratching maintains claws and marks territory, so provide sturdy posts. Balanced diets, hydration, and routine veterinary care sustain health. Safe retreats and vertical spaces reduce stress and encourage exploration." }, + { "item4", "Snowboarding across pristine slopes combines agility, balance, and speed. Riders carve smooth turns on powder, adjust stance for control, and master jumps in terrain parks. Essential gear includes boots, bindings, and helmets for safety. Embrace crisp alpine air while perfecting tricks and enjoying the thrill of winter adventure." }, + { "item5", "Urban beekeeping thrives with diverse forage across seasons. Rooftop hives benefit from trees, herbs, and staggered blooms. Provide shallow water sources and shade to counter heat stress. Prevent swarms through timely inspections and splits. Monitor mites with sugar rolls and rotate treatments. Honey reflects city terroir with surprising floral complexity." } + }; + + private AppContentIndexer? _indexer; + private CancellationTokenSource? cts = new(); + + public KnowledgeRetrieval() + { + this.InitializeComponent(); + this.Unloaded += (s, e) => + { + CleanUp(); + }; + this.Loaded += (s, e) => Page_Loaded(); // + + PopulateTextData(); + } + + protected override async Task LoadModelAsync(SampleNavigationParameters sampleParams) + { + await Task.Run(async () => + { + // Load chat client + try + { + var ragSampleParams = new SampleNavigationParameters( + sampleId: "6A526FDD-359F-4EAC-9AA6-F01DB11AE542", + modelId: "PhiSilica", + modelPath: $"file://{ModelType.PhiSilica}", + hardwareAccelerator: HardwareAccelerator.CPU, + promptTemplate: null, + sampleLoadedCompletionSource: new TaskCompletionSource(), + winMlSampleOptions: null, + loadingCanceledToken: CancellationToken.None); + + _model = await ragSampleParams.GetIChatClientAsync(); + } + catch (Exception ex) + { + ShowException(ex); + } + + // Load AppContentIndexer + var result = AppContentIndexer.GetOrCreateIndex("knowledgeRetrievalIndex"); + + if (!result.Succeeded) + { + throw new InvalidOperationException($"Failed to open index. Status = '{result.Status}', Error = '{result.ExtendedError}'"); + } + + // If result.Succeeded is true, result.Status will either be CreatedNew or OpenedExisting + if (result.Status == GetOrCreateIndexStatus.CreatedNew) + { + Debug.WriteLine("Created a new index"); + } + else if (result.Status == GetOrCreateIndexStatus.OpenedExisting) + { + Debug.WriteLine("Opened an existing index"); + } + + _indexer = result.Indexer; + await _indexer.WaitForIndexCapabilitiesAsync(); + + sampleParams.NotifyCompletion(); + }); + + IndexAll(); + } + + // + private void Page_Loaded() + { + InputBox.Focus(FocusState.Programmatic); + } + + // + protected override void OnNavigatedFrom(NavigationEventArgs e) + { + base.OnNavigatedFrom(e); + CleanUp(); + } + + private void CleanUp() + { + CancelResponse(); + _model?.Dispose(); + _indexer?.RemoveAll(); + _indexer?.Dispose(); + _indexer = null; + } + + private void CancelResponse() + { + StopBtn.Visibility = Visibility.Collapsed; + SendBtn.Visibility = Visibility.Visible; + EnableInputBoxWithPlaceholder(); + cts?.Cancel(); + cts?.Dispose(); + cts = null; + } + + private void TextBox_KeyUp(object sender, KeyRoutedEventArgs e) + { + if (e.Key == Windows.System.VirtualKey.Enter && + !Microsoft.UI.Input.InputKeyboardSource.GetKeyStateForCurrentThread(Windows.System.VirtualKey.Shift) + .HasFlag(Windows.UI.Core.CoreVirtualKeyStates.Down) && + sender is TextBox && + !string.IsNullOrWhiteSpace(InputBox.Text) && + _isImeActive == false) + { + var cursorPosition = InputBox.SelectionStart; + var text = InputBox.Text; + if (cursorPosition > 0 && (text[cursorPosition - 1] == '\n' || text[cursorPosition - 1] == '\r')) + { + text = text.Remove(cursorPosition - 1, 1); + InputBox.Text = text; + } + + InputBox.SelectionStart = cursorPosition - 1; + + SendMessage(); + } + else + { + _isImeActive = true; + } + } + + private void TextBox_PreviewKeyDown(object sender, KeyRoutedEventArgs e) + { + _isImeActive = false; + } + + private void SendMessage() + { + if (InputBox.Text.Length > 0) + { + AddMessage(InputBox.Text); + InputBox.Text = string.Empty; + SendBtn.Visibility = Visibility.Collapsed; + } + } + + private async Task> BuildContextFromUserPrompt(string queryText) + { + if (_indexer == null) + { + return new List(); + } + + var queryPrompts = await Task.Run(() => + { + // We execute a query against the index using the user's prompt string as the query text. + AppIndexQuery query = _indexer.CreateQuery(queryText); + + IReadOnlyList textMatches = query.GetNextTextMatches(5); + + List contextSnippets = new List(); + StringBuilder promptStringBuilder = new StringBuilder(); + string refIds = string.Empty; + promptStringBuilder.AppendLine("You are a helpful assistant. Please only refer to the following pieces of information when responding to the user's prompt:"); + + // For each of the matches found, we include the relevant snippets of the text files in the augmented query that we send to the language model + if (textMatches != null && textMatches.Count > 0) + { + foreach (var match in textMatches) + { + Debug.WriteLine(match.ContentId); + if (match.ContentKind == QueryMatchContentKind.AppManagedText) + { + AppManagedTextQueryMatch textResult = (AppManagedTextQueryMatch)match; + string matchingData = simpleTextData[match.ContentId]; + int offset = textResult.TextOffset; + int length = textResult.TextLength; + string matchingString; + + if (offset >= 0 && offset < matchingData.Length && length > 0 && offset + length <= matchingData.Length) + { + // Find the substring within the loaded text that contains the match: + matchingString = matchingData.Substring(offset, length); + } + else + { + matchingString = matchingData; + } + + promptStringBuilder.AppendLine(matchingString); + promptStringBuilder.AppendLine(); + + refIds += string.IsNullOrEmpty(refIds) ? match.ContentId : ", " + match.ContentId; + } + } + } + + promptStringBuilder.AppendLine("Please provide a short response of less than 50 words to the following user prompt:"); + promptStringBuilder.AppendLine(queryText); + + contextSnippets.Add(refIds); + contextSnippets.Add(promptStringBuilder.ToString()); + + return contextSnippets; + }); + + return queryPrompts; + } + + private void AddMessage(string text) + { + if (_model == null) + { + return; + } + + Messages.Add(new Message(text.Trim(), DateTime.Now, ChatRole.User)); + var contentStartedBeingGenerated = false; // + NarratorHelper.Announce(InputBox, "Generating response, please wait.", "ChatWaitAnnouncementActivityId"); // > + SendSampleInteractedEvent("AddMessage"); // + + Task.Run(async () => + { + var history = Messages.Select(m => new ChatMessage(m.Role, m.Content)).ToList(); + + var responseMessage = new Message(string.Empty, DateTime.Now, ChatRole.Assistant) + { + IsPending = true + }; + + DispatcherQueue.TryEnqueue(() => + { + Messages.Add(responseMessage); + StopBtn.Visibility = Visibility.Visible; + InputBox.IsEnabled = false; + }); + + cts = new CancellationTokenSource(); + + // Use AppContentIndexer query here. + var userPrompt = await BuildContextFromUserPrompt(text); + + history.Insert(0, new ChatMessage(ChatRole.System, userPrompt[1])); + + // + ShowDebugInfo(null); + var swEnd = Stopwatch.StartNew(); + var swTtft = Stopwatch.StartNew(); + int outputTokens = 0; + + // + int currentThinkTagIndex = -1; // -1 means not inside any think/auxiliary section + string rolling = string.Empty; + + await foreach (var messagePart in _model.GetStreamingResponseAsync(history, null, cts.Token)) + { + // + if (outputTokens == 0) + { + swTtft.Stop(); + } + + outputTokens++; + double currentTps = outputTokens / Math.Max(swEnd.Elapsed.TotalSeconds - swTtft.Elapsed.TotalSeconds, 1e-6); + ShowDebugInfo($"{Math.Round(currentTps)} tokens per second\n{outputTokens} tokens used\n{swTtft.Elapsed.TotalSeconds:0.00}s to first token\n{swEnd.Elapsed.TotalSeconds:0.00}s total"); + + // + var part = messagePart; + + DispatcherQueue.TryEnqueue(() => + { + if (responseMessage.IsPending) + { + responseMessage.IsPending = false; + } + + // Parse character by character/fragment to identify think tags (e.g., ..., ...) + rolling += part; + + while (!string.IsNullOrEmpty(rolling)) + { + if (currentThinkTagIndex == -1) + { + // Find the earliest occurring open marker among supported think tags + int earliestIdx = -1; + int foundTagIndex = -1; + for (int i = 0; i < ThinkTagOpens.Length; i++) + { + int idx = rolling.IndexOf(ThinkTagOpens[i], StringComparison.Ordinal); + if (idx >= 0 && (earliestIdx == -1 || idx < earliestIdx)) + { + earliestIdx = idx; + foundTagIndex = i; + } + } + + if (earliestIdx >= 0) + { + // Output safe content before the start marker + if (earliestIdx > 0) + { + responseMessage.Content = string.Concat(responseMessage.Content, rolling.AsSpan(0, earliestIdx)); + } + + // Enter think mode, discard the marker text itself + rolling = rolling.Substring(earliestIdx + ThinkTagOpens[foundTagIndex].Length); + currentThinkTagIndex = foundTagIndex; + continue; + } + else + { + // Start marker not found: only flush safe parts, keep the tail that might form a marker + int keep = MaxOpenThinkMarkerLength - 1; + if (rolling.Length > keep) + { + int flushLen = rolling.Length - keep; + responseMessage.Content = string.Concat(responseMessage.Content.TrimStart(), rolling.AsSpan(0, flushLen)); + rolling = rolling.Substring(flushLen); + } + + break; + } + } + else + { + string closeMarker = ThinkTagCloses[currentThinkTagIndex]; + int closeIdx = rolling.IndexOf(closeMarker, StringComparison.Ordinal); + if (closeIdx >= 0) + { + // Append content before the closing marker to the think box + if (closeIdx > 0) + { + responseMessage.ThinkContent = string.Concat(responseMessage.ThinkContent, rolling.AsSpan(0, closeIdx)); + } + + // Exit think mode, discard the closing marker + rolling = rolling.Substring(closeIdx + closeMarker.Length); + currentThinkTagIndex = -1; + continue; + } + else + { + // Closing marker not found: only flush safe parts, keep the tail that might form a marker + int keep = closeMarker.Length - 1; + if (rolling.Length > keep) + { + int flushLen = rolling.Length - keep; + responseMessage.ThinkContent = string.Concat(responseMessage.ThinkContent, rolling.AsSpan(0, flushLen)); + rolling = rolling.Substring(flushLen); + } + + break; + } + } + } + + // + if (!contentStartedBeingGenerated) + { + NarratorHelper.Announce(InputBox, "Response has started generating.", "ChatResponseAnnouncementActivityId"); + contentStartedBeingGenerated = true; + } + + // + }); + } + + // Flush remaining tail content (if any) + DispatcherQueue.TryEnqueue(() => + { + responseMessage.IsPending = false; + if (!string.IsNullOrEmpty(rolling)) + { + if (currentThinkTagIndex != -1) + { + responseMessage.ThinkContent += rolling; + } + else + { + responseMessage.Content = responseMessage.Content.TrimStart() + rolling; + } + } + + responseMessage.Content += "\n\n" + "Referenced items: " + userPrompt[0]; + }); + + // + swEnd.Stop(); + double tps = outputTokens / Math.Max(swEnd.Elapsed.TotalSeconds - swTtft.Elapsed.TotalSeconds, 1e-6); + ShowDebugInfo($"{Math.Round(tps)} tokens per second\n{outputTokens} tokens used\n{swTtft.Elapsed.TotalSeconds:0.00}s to first token\n{swEnd.Elapsed.TotalSeconds:0.00}s total"); + + // + cts?.Dispose(); + cts = null; + + DispatcherQueue.TryEnqueue(() => + { + NarratorHelper.Announce(InputBox, "Content has finished generating.", "ChatDoneAnnouncementActivityId"); // + StopBtn.Visibility = Visibility.Collapsed; + SendBtn.Visibility = Visibility.Visible; + EnableInputBoxWithPlaceholder(); + }); + }); + } + + private void SendBtn_Click(object sender, RoutedEventArgs e) + { + SendMessage(); + } + + private void StopBtn_Click(object sender, RoutedEventArgs e) + { + CancelResponse(); + } + + private void InputBox_TextChanged(object sender, TextChangedEventArgs e) + { + SendBtn.IsEnabled = !string.IsNullOrWhiteSpace(InputBox.Text); + } + + private void EnableInputBoxWithPlaceholder() + { + InputBox.IsEnabled = true; + } + + private void InvertedListView_Loaded(object sender, RoutedEventArgs e) + { + _scrollViewer = FindElement(InvertedListView); + + ItemsStackPanel? itemsStackPanel = FindElement(InvertedListView); + if (itemsStackPanel != null) + { + itemsStackPanel.SizeChanged += ItemsStackPanel_SizeChanged; + } + } + + private void ItemsStackPanel_SizeChanged(object sender, SizeChangedEventArgs e) + { + if (_scrollViewer != null) + { + bool isScrollbarVisible = _scrollViewer.ComputedVerticalScrollBarVisibility == Visibility.Visible; + + if (isScrollbarVisible) + { + InvertedListView.Padding = new Thickness(-12, 0, 12, 24); + } + else + { + InvertedListView.Padding = new Thickness(-12, 0, -12, 24); + } + } + } + + private T? FindElement(DependencyObject element) + where T : DependencyObject + { + if (element is T targetElement) + { + return targetElement; + } + + for (int i = 0; i < VisualTreeHelper.GetChildrenCount(element); i++) + { + var child = VisualTreeHelper.GetChild(element, i); + var result = FindElement(child); + if (result != null) + { + return result; + } + } + + return null; + } + + private async void SemanticTextBox_TextChanged(object sender, TextChangedEventArgs e) + { + if (sender is TextBox textBox) + { + string? id = textBox.Tag as string; + string value = textBox.Text; + + // Update local dictionary and observable collection + var item = TextDataItems.FirstOrDefault(x => x.Id == id); + if (item != null) + { + item.Value = value; + } + + if (id != null) + { + if (simpleTextData.ContainsKey(id)) + { + simpleTextData[id] = value; + } + + IndexingMessage.IsOpen = true; + await Task.Run(() => + { + IndexTextData(id, value); + }); + } + + IndexingMessage.IsOpen = false; + } + } + + private async void AddTextDataButton_Click(object sender, RoutedEventArgs e) + { + // Find the lowest unused id in the form itemN + int nextIndex = 1; + string newId; + var existingIds = new HashSet(simpleTextData.Keys.Concat(TextDataItems.Select(x => x.Id).Where(id => id != null)).Cast()); + do + { + newId = $"item{nextIndex}"; + nextIndex++; + } + while (existingIds.Contains(newId)); + + string defaultValue = "New item text..."; + + // Add to dictionary + simpleTextData[newId] = defaultValue; + + // Add to observable collection + var newItem = new TextDataItem { Id = newId, Value = defaultValue }; + TextDataItems.Add(newItem); + + IndexingMessage.IsOpen = true; + await Task.Run(() => + { + IndexTextData(newId, defaultValue); + }); + IndexingMessage.IsOpen = false; + } + + private async void CloseButton_Click(object sender, RoutedEventArgs e) + { + if (sender is Button button) + { + if (button.Tag is TextDataItem textItem) + { + TextDataItems.Remove(textItem); + + if (!string.IsNullOrEmpty(textItem.Id)) + { + if (simpleTextData.ContainsKey(textItem.Id)) + { + simpleTextData.Remove(textItem.Id); + } + + RemovedItemMessage.IsOpen = true; + RemovedItemMessage.Message = $"Removed {textItem.Id} from index"; + await Task.Run(() => + { + RemoveItemFromIndex(textItem.Id); + }); + } + + RemovedItemMessage.IsOpen = false; + } + } + } + + private void RemoveItemFromIndex(string id) + { + if (_indexer == null) + { + return; + } + + // Remove item from index + _indexer.Remove(id); + } + + private void IndexTextData(string id, string value) + { + if (_indexer == null) + { + return; + } + + // Index Textbox content + IndexableAppContent textContent = AppManagedIndexableAppContent.CreateFromString(id, value); + _indexer.AddOrUpdate(textContent); + } + + private async void IndexAll() + { + IndexingMessage.IsOpen = true; + + await Task.Run(() => + { + foreach (var kvp in simpleTextData) + { + IndexTextData(kvp.Key, kvp.Value); + } + }); + + IndexingMessage.IsOpen = false; + } + + private void IndexAllButton_Click(object sender, RoutedEventArgs e) + { + IndexAll(); + } + + private async Task LoadBitmap(string uriString) + { + try + { + StorageFile file; + if (uriString.StartsWith("ms-appx", StringComparison.OrdinalIgnoreCase)) + { + file = await StorageFile.GetFileFromApplicationUriAsync(new Uri(uriString)); + } + else + { + // Assume it's a file path for user-uploaded images + file = await StorageFile.GetFileFromPathAsync(uriString); + } + + using var stream = await file.OpenAsync(FileAccessMode.Read); + var decoder = await BitmapDecoder.CreateAsync(stream); + + return await decoder.GetSoftwareBitmapAsync(BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied); + } + catch (Exception ex) + { + System.Diagnostics.Debug.WriteLine($"Error loading image: {ex.Message}"); + } + + return null; + } + + private void PopulateTextData() + { + foreach (var kvp in simpleTextData) + { + TextDataItems.Add(new TextDataItem { Id = kvp.Key, Value = kvp.Value }); + } + } + + private CancellationToken CancelGenerationAndGetNewToken() + { + cts?.Cancel(); + cts?.Dispose(); + cts = new CancellationTokenSource(); + return cts.Token; + } +} \ No newline at end of file diff --git a/AIDevGallery/Samples/WCRAPIs/SemanticSearch.xaml b/AIDevGallery/Samples/WCRAPIs/SemanticSearch.xaml new file mode 100644 index 00000000..1f7285d1 --- /dev/null +++ b/AIDevGallery/Samples/WCRAPIs/SemanticSearch.xaml @@ -0,0 +1,264 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/AIDevGallery/Samples/WCRAPIs/SemanticSearch.xaml.cs b/AIDevGallery/Samples/WCRAPIs/SemanticSearch.xaml.cs new file mode 100644 index 00000000..66abf35a --- /dev/null +++ b/AIDevGallery/Samples/WCRAPIs/SemanticSearch.xaml.cs @@ -0,0 +1,658 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using AIDevGallery.Models; +using AIDevGallery.Samples.Attributes; +using AIDevGallery.Samples.SharedCode; +using Microsoft.UI.Xaml; +using Microsoft.UI.Xaml.Controls; +using Microsoft.UI.Xaml.Media.Imaging; +using Microsoft.UI.Xaml.Navigation; +using Microsoft.Windows.AI.Search.Experimental.AppContentIndex; +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Windows.Graphics.Imaging; +using Windows.Storage; +using Windows.Storage.Pickers; + +namespace AIDevGallery.Samples.WCRAPIs; + +[GallerySample( + Name = "Semantic Search", + Model1Types = [ModelType.SemanticSearch], + Scenario = ScenarioType.TextSemanticSearch, + Id = "f8465a45-8e23-4485-8c16-9909e96eacf6", + SharedCode = [ + SharedCodeEnum.DataItems + ], + AssetFilenames = [ + "InteriorDesign.png", + "TofuBowlRecipe.png", + "ShakshukaRecipe.png" + ], + NugetPackageReferences = [ + "Microsoft.Extensions.AI" + ], + Icon = "\uEE6F")] + +internal sealed partial class SemanticSearch : BaseSamplePage +{ + private ObservableCollection TextDataItems { get; } = new(); + private ObservableCollection ImageDataItems { get; } = new(); + + // This is some text data that we want to add to the index: + private Dictionary simpleTextData = new Dictionary + { + { "item1", "Preparing a hearty vegetable stew begins with chopping fresh carrots, onions, and celery. Sauté them in olive oil until fragrant, then add diced tomatoes, herbs, and vegetable broth. Simmer gently for an hour, allowing flavors to meld into a comforting dish perfect for cold evenings." }, + { "item2", "Modern exhibition design combines narrative flow with spatial strategy. Lighting emphasizes focal objects while circulation paths avoid bottlenecks. Materials complement artifacts without visual competition. Interactive elements invite engagement but remain intuitive. Environmental controls protect sensitive works. Success balances scholarship, aesthetics, and visitor experience through thoughtful, cohesive design choices." }, + { "item3", "Domestic cats communicate through posture, tail flicks, and vocalizations. Play mimics hunting behaviors like stalking and pouncing, supporting agility and mental stimulation. Scratching maintains claws and marks territory, so provide sturdy posts. Balanced diets, hydration, and routine veterinary care sustain health. Safe retreats and vertical spaces reduce stress and encourage exploration." }, + { "item4", "Snowboarding across pristine slopes combines agility, balance, and speed. Riders carve smooth turns on powder, adjust stance for control, and master jumps in terrain parks. Essential gear includes boots, bindings, and helmets for safety. Embrace crisp alpine air while perfecting tricks and enjoying the thrill of winter adventure." }, + { "item5", "Urban beekeeping thrives with diverse forage across seasons. Rooftop hives benefit from trees, herbs, and staggered blooms. Provide shallow water sources and shade to counter heat stress. Prevent swarms through timely inspections and splits. Monitor mites with sugar rolls and rotate treatments. Honey reflects city terroir with surprising floral complexity." } + }; + + private Dictionary simpleImageData = new Dictionary + { + { "image1", "InteriorDesign.png" }, + { "image2", "TofuBowlRecipe.png" }, + { "image3", "ShakshukaRecipe.png" }, + }; + + private AppContentIndexer? _indexer; + private CancellationTokenSource cts = new(); + + public SemanticSearch() + { + this.InitializeComponent(); + this.Unloaded += (s, e) => + { + CleanUp(); + }; + this.Loaded += (s, e) => Page_Loaded(); // + + PopulateTextData(); + PopulateImageData(); + } + + protected override async Task LoadModelAsync(SampleNavigationParameters sampleParams) + { + await Task.Run(async () => + { + var result = AppContentIndexer.GetOrCreateIndex("semanticSearchIndex"); + + if (!result.Succeeded) + { + ShowException(null, $"Failed to open index. Status = '{result.Status}', Error = '{result.ExtendedError}'"); + return; + } + + // If result.Succeeded is true, result.Status will either be CreatedNew or OpenedExisting + if (result.Status == GetOrCreateIndexStatus.CreatedNew) + { + Debug.WriteLine("Created a new index"); + } + else if (result.Status == GetOrCreateIndexStatus.OpenedExisting) + { + Debug.WriteLine("Opened an existing index"); + } + + _indexer = result.Indexer; + await _indexer.WaitForIndexCapabilitiesAsync(); + + _indexer.Listener.IndexCapabilitiesChanged += Listener_IndexCapabilitiesChanged; + LoadAppIndexCapabilities(); + + sampleParams.NotifyCompletion(); + }); + + IndexAll(); + } + + // + private void Page_Loaded() + { + textDataItemsView.Focus(FocusState.Programmatic); + } + + // + protected override void OnNavigatedFrom(NavigationEventArgs e) + { + base.OnNavigatedFrom(e); + CleanUp(); + } + + private void CleanUp() + { + _indexer?.RemoveAll(); + _indexer?.Dispose(); + _indexer = null; + } + + // Update and index local test text data on TextBox text changed + private async void SemanticTextBox_TextChanged(object sender, TextChangedEventArgs e) + { + if (sender is TextBox textBox) + { + string? id = textBox.Tag as string; + string value = textBox.Text; + + if (id != null) + { + // Update local dictionary and observable collection + var item = TextDataItems.FirstOrDefault(x => x.Id == id); + if (item != null) + { + item.Value = value; + } + + if (simpleTextData.ContainsKey(id)) + { + simpleTextData[id] = value; + } + + // Index text data + IndexingMessage.IsOpen = true; + await Task.Run(async () => + { + IndexTextData(id, value); + var isIdle = await _indexer?.WaitForIndexingIdleAsync(50000); + }); + } + + IndexingMessage.IsOpen = false; + } + } + + // Update and index local test image data on image opened + private async void ImageData_ImageOpened(object sender, RoutedEventArgs e) + { + if (sender is Microsoft.UI.Xaml.Controls.Image image) + { + string? id = image.Tag as string; + string uriString = string.Empty; + string fileName = string.Empty; + + if (image.Source is BitmapImage bitmapImage && bitmapImage.UriSource != null) + { + uriString = bitmapImage.UriSource.ToString(); + } + + SoftwareBitmap? bitmap = null; + if (!string.IsNullOrEmpty(uriString)) + { + bitmap = await LoadBitmap(uriString); + } + + if (id != null && bitmap != null) + { + // Update local dictionary and observable collection + var item = ImageDataItems.FirstOrDefault(x => x.Id == id); + if (item != null) + { + fileName = Path.GetFileName(uriString); + item.ImageSource = fileName; + } + + string imageVal = uriString.StartsWith("ms-appx", StringComparison.OrdinalIgnoreCase) ? fileName : uriString; + + if (!simpleImageData.TryAdd(id, imageVal)) + { + simpleImageData[id] = imageVal; + } + + IndexingMessage.IsOpen = true; + await Task.Run(async () => + { + IndexImageData(id, bitmap); + var isIdle = await _indexer?.WaitForIndexingIdleAsync(50000); + }); + } + + IndexingMessage.IsOpen = false; + } + } + + private void SearchBox_QuerySubmitted(AutoSuggestBox sender, AutoSuggestBoxQuerySubmittedEventArgs args) + { + string searchText = SearchBox.Text; + if (string.IsNullOrWhiteSpace(searchText)) + { + Debug.WriteLine("Search text is empty."); + return; + } + + if (_indexer == null) + { + ResultStatusTextBlock.Text = "Indexer is unavailable."; + return; + } + + ResultsGrid.Visibility = Visibility.Visible; + ResultStatusTextBlock.Text = "Searching..."; + + // Create query options + AppIndexQueryOptions queryOptions = new AppIndexQueryOptions(); + + // Set language if provided + string queryLanguage = QueryLanguageTextBox.Text; + if (!string.IsNullOrWhiteSpace(queryLanguage)) + { + queryOptions.Language = queryLanguage; + } + + // Create text match options + TextMatchOptions textMatchOptions = new TextMatchOptions + { + MatchScope = (QueryMatchScope)TextMatchScopeComboBox.SelectedIndex, + TextMatchType = (TextLexicalMatchType)TextMatchTypeComboBox.SelectedIndex + }; + + // Create image match options + ImageMatchOptions imageMatchOptions = new ImageMatchOptions + { + MatchScope = (QueryMatchScope)ImageMatchScopeComboBox.SelectedIndex, + ImageOcrTextMatchType = (TextLexicalMatchType)ImageOcrTextMatchTypeComboBox.SelectedIndex + }; + + CancellationToken ct = CancelGenerationAndGetNewToken(); + + string textResults = string.Empty; + var imageResults = new List(); + + Task.Run( + () => + { + // Create query + AppIndexQuery query = _indexer.CreateQuery(searchText, queryOptions); + + // Get text matches + IReadOnlyList textMatches = query.GetNextTextMatches(5); + + if (textMatches != null && textMatches.Count > 0) + { + foreach (var match in textMatches) + { + Debug.WriteLine(match.ContentId); + if (match.ContentKind == QueryMatchContentKind.AppManagedText) + { + AppManagedTextQueryMatch textResult = (AppManagedTextQueryMatch)match; + string matchingData = simpleTextData[match.ContentId]; + int offset = textResult.TextOffset; + int length = textResult.TextLength; + string matchingString = matchingData.Substring(offset, length); + textResults += matchingString + "\n\n"; + } + } + } + + // Get image matches + IReadOnlyList imageMatches = query.GetNextImageMatches(5); + + if (imageMatches != null && imageMatches.Count > 0) + { + foreach (var match in imageMatches) + { + Debug.WriteLine(match.ContentId); + if (match.ContentKind == QueryMatchContentKind.AppManagedImage) + { + AppManagedImageQueryMatch imageResult = (AppManagedImageQueryMatch)match; + + if (simpleImageData.TryGetValue(imageResult.ContentId, out var imagePath)) + { + string imageVal = imagePath.StartsWith("file://", StringComparison.OrdinalIgnoreCase) ? imagePath : $"ms-appx:///Assets/{imagePath}"; + imageResults.Add(imageVal); + } + } + } + } + + DispatcherQueue.TryEnqueue(() => + { + if ((textMatches == null || textMatches.Count == 0) && (imageResults == null || imageResults.Count == 0)) + { + ResultStatusTextBlock.Text = "No results found."; + } + else + { + ResultStatusTextBlock.Text = "Search Results:"; + } + + if (textMatches != null && textMatches.Count > 0) + { + ResultsTextBlock.Visibility = Visibility.Visible; + ResultsTextBlock.Text = textResults; + } + else + { + ResultsTextBlock.Visibility = Visibility.Collapsed; + } + + if (imageResults != null && imageResults.Count > 0) + { + ImageResultsBox.ItemsSource = imageResults; + ImageResultsBox.Visibility = Visibility.Visible; + } + else + { + ImageResultsBox.ItemsSource = null; + ImageResultsBox.Visibility = Visibility.Collapsed; + } + }); + }, + ct); + } + + private async void AddTextDataButton_Click(object sender, RoutedEventArgs e) + { + // Find the lowest unused id in the form itemN + int nextIndex = 1; + string newId; + var existingIds = new HashSet(simpleTextData.Keys.Concat(TextDataItems.Select(x => x.Id ?? string.Empty)).Where(id => !string.IsNullOrEmpty(id))); + do + { + newId = $"item{nextIndex}"; + nextIndex++; + } + while (existingIds.Contains(newId)); + + string defaultValue = "New item text..."; + + // Add to dictionary + simpleTextData[newId] = defaultValue; + + // Add to observable collection + var newItem = new TextDataItem { Id = newId, Value = defaultValue }; + TextDataItems.Add(newItem); + + IndexingMessage.IsOpen = true; + await Task.Run(() => + { + IndexTextData(newId, defaultValue); + }); + IndexingMessage.IsOpen = false; + } + + private async void CloseButton_Click(object sender, RoutedEventArgs e) + { + if (sender is Button button) + { + if (button.Tag is TextDataItem textItem) + { + TextDataItems.Remove(textItem); + + if (!string.IsNullOrEmpty(textItem.Id)) + { + if (simpleTextData.ContainsKey(textItem.Id)) + { + simpleTextData.Remove(textItem.Id); + } + + RemovedItemMessage.IsOpen = true; + RemovedItemMessage.Message = $"Removed {textItem.Id} from index"; + await Task.Run(() => + { + RemoveItemFromIndex(textItem.Id); + }); + } + + RemovedItemMessage.IsOpen = false; + } + else if (button.Tag is ImageDataItem imageItem) + { + ImageDataItems.Remove(imageItem); + + if (!string.IsNullOrEmpty(imageItem.Id)) + { + if (simpleImageData.ContainsKey(imageItem.Id)) + { + simpleImageData.Remove(imageItem.Id); + } + + RemovedItemMessage.IsOpen = true; + RemovedItemMessage.Message = $"Removed {imageItem.Id} from index"; + await Task.Run(() => + { + RemoveItemFromIndex(imageItem.Id); + }); + } + + RemovedItemMessage.IsOpen = false; + } + } + } + + private async void UploadImageButton_Click(object sender, RoutedEventArgs e) + { + SendSampleInteractedEvent("LoadImageClicked"); + var window = new Window(); + var hwnd = WinRT.Interop.WindowNative.GetWindowHandle(window); + + var picker = new FileOpenPicker(); + + WinRT.Interop.InitializeWithWindow.Initialize(picker, hwnd); + + picker.FileTypeFilter.Add(".png"); + picker.FileTypeFilter.Add(".jpeg"); + picker.FileTypeFilter.Add(".jpg"); + + picker.ViewMode = PickerViewMode.Thumbnail; + + StorageFile file = await picker.PickSingleFileAsync(); + if (file != null) + { + // Generate a unique id for the new image + int nextIndex = 1; + string newId; + do + { + newId = $"image{ImageDataItems.Count + nextIndex}"; + nextIndex++; + } + while (ImageDataItems.Any(i => i.Id == newId)); + + // Create a ms-appx URI for the image (or use file path for local images) + var imageUri = file.Path; + + // Add to collection and dictionary + ImageDataItems.Add(new ImageDataItem { Id = newId, ImageSource = imageUri }); + simpleImageData[newId] = imageUri; + } + } + + private void RemoveItemFromIndex(string id) + { + if (_indexer == null) + { + return; + } + + // Remove item from index + _indexer.Remove(id); + } + + private void IndexTextData(string id, string value) + { + if (_indexer == null) + { + return; + } + + // Index Textbox content + IndexableAppContent textContent = AppManagedIndexableAppContent.CreateFromString(id, value); + _indexer.AddOrUpdate(textContent); + } + + private void IndexImageData(string id, SoftwareBitmap bitmap) + { + if (_indexer == null) + { + return; + } + + // Index image content + IndexableAppContent imageContent = AppManagedIndexableAppContent.CreateFromBitmap(id, bitmap); + _indexer.AddOrUpdate(imageContent); + } + + private async void IndexAll() + { + IndexingMessage.IsOpen = true; + + await Task.Run(async () => + { + foreach (var kvp in simpleTextData) + { + IndexTextData(kvp.Key, kvp.Value); + } + + foreach (var kvp in simpleImageData) + { + var uri = $"ms-appx:///Assets/{kvp.Value}"; + SoftwareBitmap? bitmap = await LoadBitmap(uri); + if (bitmap != null) + { + IndexImageData(kvp.Key, bitmap); + } + } + + var isIdle = await _indexer?.WaitForIndexingIdleAsync(50000); + }); + + IndexingMessage.IsOpen = false; + } + + private void IndexAllButton_Click(object sender, RoutedEventArgs e) + { + IndexAll(); + } + + private async void LoadAppIndexCapabilities() + { + if (_indexer == null) + { + return; + } + + IndexCapabilities capabilities = await Task.Run(() => + { + return _indexer.GetIndexCapabilities(); + }); + + DispatcherQueue.TryEnqueue(() => + { + bool textLexicalAvailable = + capabilities.GetCapabilityState(IndexCapability.TextLexical).InitializationStatus == IndexCapabilityInitializationStatus.Initialized; + bool textSemanticAvailable = + capabilities.GetCapabilityState(IndexCapability.TextSemantic).InitializationStatus == IndexCapabilityInitializationStatus.Initialized; + bool imageSemanticAvailable = + capabilities.GetCapabilityState(IndexCapability.ImageSemantic).InitializationStatus == IndexCapabilityInitializationStatus.Initialized; + bool imageOcrAvailable = + capabilities.GetCapabilityState(IndexCapability.ImageOcr).InitializationStatus == IndexCapabilityInitializationStatus.Initialized; + + // Disable text sample if both text capabilities are unavailable + textDataItemsView.IsEnabled = textLexicalAvailable || textSemanticAvailable; + uploadTextButton.IsEnabled = imageSemanticAvailable || imageOcrAvailable; + + // Disable image sample if both image capabilities are unavailable + ImageDataItemsView.IsEnabled = imageSemanticAvailable || imageOcrAvailable; + uploadImageButton.IsEnabled = imageSemanticAvailable || imageOcrAvailable; + + var unavailable = new List(); + if (!textLexicalAvailable) + { + unavailable.Add("TextLexical"); + } + + if (!textSemanticAvailable) + { + unavailable.Add("TextSemantic"); + } + + if (!imageSemanticAvailable) + { + unavailable.Add("ImageSemantic"); + } + + if (!imageOcrAvailable) + { + unavailable.Add("ImageOcr"); + } + + if (unavailable.Count > 0) + { + IndexCapabilitiesMessage.Message = $"Unavailable: {string.Join(", ", unavailable)}"; + IndexCapabilitiesMessage.IsOpen = true; + AllIndexAvailableTextBlock.Visibility = Visibility.Collapsed; + } + else + { + // All capabilities are available + IndexCapabilitiesMessage.IsOpen = false; + AllIndexAvailableTextBlock.Visibility = Visibility.Visible; + } + }); + } + + private void Listener_IndexCapabilitiesChanged(AppContentIndexer indexer, IndexCapabilities statusResult) + { + LoadAppIndexCapabilities(); + } + + private async Task LoadBitmap(string uriString) + { + try + { + StorageFile file; + if (uriString.StartsWith("ms-appx", StringComparison.OrdinalIgnoreCase)) + { + file = await StorageFile.GetFileFromApplicationUriAsync(new Uri(uriString)); + } + else + { + // Assume it's a file path for user-uploaded images + file = await StorageFile.GetFileFromPathAsync(uriString); + } + + using var stream = await file.OpenAsync(FileAccessMode.Read); + var decoder = await BitmapDecoder.CreateAsync(stream); + + return await decoder.GetSoftwareBitmapAsync(BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied); + } + catch (Exception ex) + { + System.Diagnostics.Debug.WriteLine($"Error loading image: {ex.Message}"); + } + + return null; + } + + private void PopulateTextData() + { + foreach (var kvp in simpleTextData) + { + TextDataItems.Add(new TextDataItem { Id = kvp.Key, Value = kvp.Value }); + } + } + + private void PopulateImageData() + { + foreach (var kvp in simpleImageData) + { + var uri = $"ms-appx:///Assets/{kvp.Value}"; + ImageDataItems.Add(new ImageDataItem { Id = kvp.Key, ImageSource = uri }); + } + } + + private CancellationToken CancelGenerationAndGetNewToken() + { + cts.Cancel(); + cts.Dispose(); + cts = new CancellationTokenSource(); + return cts.Token; + } +} \ No newline at end of file diff --git a/AIDevGallery/Styles/Card.xaml b/AIDevGallery/Styles/Card.xaml index 3188734e..84d66d7e 100644 --- a/AIDevGallery/Styles/Card.xaml +++ b/AIDevGallery/Styles/Card.xaml @@ -36,4 +36,22 @@ + + + diff --git a/AIDevGallery/Utils/AppData.cs b/AIDevGallery/Utils/AppData.cs index 78692a50..82ac04a2 100644 --- a/AIDevGallery/Utils/AppData.cs +++ b/AIDevGallery/Utils/AppData.cs @@ -3,6 +3,7 @@ using AIDevGallery.Models; using AIDevGallery.Telemetry; +using CommunityToolkit.Mvvm.ComponentModel; using Microsoft.ML.OnnxRuntime; using Microsoft.Windows.AI.ContentSafety; using System; @@ -16,7 +17,7 @@ namespace AIDevGallery.Utils; -internal class AppData +internal partial class AppData : ObservableObject { private static readonly SemaphoreSlim _saveSemaphore = new(1, 1); @@ -31,6 +32,10 @@ internal class AppData public bool IsFirstRun { get; set; } public bool IsDiagnosticsMessageDismissed { get; set; } + + [ObservableProperty] + public partial bool IsAppContentSearchEnabled { get; set; } + public bool IsAppContentIndexCompleted { get; set; } public Dictionary>? ModelTypeToUserAddedModelsMapping { get; set; } public string LastAdapterPath { get; set; } @@ -46,6 +51,7 @@ public AppData() IsDiagnosticDataEnabled = !PrivacyConsentHelpers.IsPrivacySensitiveRegion(); IsFirstRun = true; IsDiagnosticsMessageDismissed = false; + IsAppContentSearchEnabled = false; LastAdapterPath = string.Empty; LastSystemPrompt = string.Empty; WinMLSampleOptions = new WinMlSampleOptions(ExecutionProviderDevicePolicy.DEFAULT, null, false, null); diff --git a/AIDevGallery/Utils/AppUtils.cs b/AIDevGallery/Utils/AppUtils.cs index 1461d506..2389df5e 100644 --- a/AIDevGallery/Utils/AppUtils.cs +++ b/AIDevGallery/Utils/AppUtils.cs @@ -175,6 +175,7 @@ public static string GetHardwareAcceleratorString(HardwareAccelerator hardwareAc case HardwareAccelerator.QNN: case HardwareAccelerator.NPU: return "NPU"; + case HardwareAccelerator.ACI: case HardwareAccelerator.WCRAPI: return "Windows AI API"; default: diff --git a/Directory.Packages.props b/Directory.Packages.props index 12b692ab..a8e7a60a 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -29,7 +29,7 @@ - +