diff --git a/dotnet/CoreLib/Memory.cs b/dotnet/CoreLib/Memory.cs index 7f08a2653..9181c2409 100644 --- a/dotnet/CoreLib/Memory.cs +++ b/dotnet/CoreLib/Memory.cs @@ -204,4 +204,21 @@ public Task AskAsync( index = IndexExtensions.CleanName(index); return this._searchClient.AskAsync(index: index, question: question, filters: filters, cancellationToken: cancellationToken); } + + public Task ListAsync( + string? index = null, + MemoryFilter? filter = null, + ICollection? filters = null, + CancellationToken cancellationToken = default) + { + if (filter != null) + { + if (filters == null) { filters = new List(); } + + filters.Add(filter); + } + + index = IndexExtensions.CleanName(index); + return this._searchClient.ListAsync(index: index, filters: filters, cancellationToken: cancellationToken); + } } diff --git a/dotnet/CoreLib/Search/SearchClient.cs b/dotnet/CoreLib/Search/SearchClient.cs index b2fe2da99..2a6f9df25 100644 --- a/dotnet/CoreLib/Search/SearchClient.cs +++ b/dotnet/CoreLib/Search/SearchClient.cs @@ -288,6 +288,76 @@ public async Task AskAsync( return answer; } + public async Task ListAsync( + string index, + ICollection? filters = null, + CancellationToken cancellationToken = default) + { + var result = new SearchResult + { + Query = string.Empty, + Results = new List() + }; + + this._log.LogTrace("Fetching all memories"); + IAsyncEnumerable matches = this._vectorDb.GetListAsync( + indexName: index, limit: -1, filters: filters, cancellationToken: cancellationToken); + + // Memories are sorted by relevance, starting from the most relevant + await foreach (MemoryRecord memory in matches.WithCancellation(cancellationToken)) + { + // Note: a document can be composed by multiple files + string documentId = memory.Tags[Constants.ReservedDocumentIdTag].FirstOrDefault() ?? string.Empty; + + // Identify the file in case there are multiple files + string fileId = memory.Tags[Constants.ReservedFileIdTag].FirstOrDefault() ?? string.Empty; + + // TODO: URL to access the file + string linkToFile = $"{documentId}/{fileId}"; + + string fileContentType = memory.Tags[Constants.ReservedFileTypeTag].FirstOrDefault() ?? string.Empty; + string fileName = memory.Payload[Constants.ReservedPayloadFileNameField].ToString() ?? string.Empty; + + var partitionText = memory.Payload[Constants.ReservedPayloadTextField].ToString()?.Trim() ?? ""; + if (string.IsNullOrEmpty(partitionText)) + { + this._log.LogError("The document partition is empty, doc: {0}", memory.Id); + continue; + } + + // If the file is already in the list of citations, only add the partition + var citation = result.Results.FirstOrDefault(x => x.Link == linkToFile); + if (citation == null) + { + citation = new Citation(); + result.Results.Add(citation); + } + + // Add the partition to the list of citations + citation.Link = linkToFile; + citation.SourceContentType = fileContentType; + citation.SourceName = fileName; + citation.Tags = memory.Tags; + +#pragma warning disable CA1806 // it's ok if parsing fails + DateTimeOffset.TryParse(memory.Payload[Constants.ReservedPayloadLastUpdateField].ToString(), out var lastUpdate); +#pragma warning restore CA1806 + + citation.Partitions.Add(new Citation.Partition + { + Text = partitionText, + LastUpdate = lastUpdate, + }); + } + + if (result.Results.Count == 0) + { + this._log.LogDebug("No memories found"); + } + + return result; + } + private async Task GenerateEmbeddingAsync(string text) { this._log.LogTrace("Generating embedding for the query"); diff --git a/examples/001-dotnet-Serverless/Program.cs b/examples/001-dotnet-Serverless/Program.cs index fb6cc629c..2254d9f53 100644 --- a/examples/001-dotnet-Serverless/Program.cs +++ b/examples/001-dotnet-Serverless/Program.cs @@ -23,6 +23,7 @@ bool useImages = false; // Enable Azure Form Recognizer OCR to use this bool retrieval = true; bool purge = true; +bool listContents = true; // ======================= // === INGESTION ========= @@ -189,6 +190,25 @@ await memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/sem Console.WriteLine($"\nNews: {answer.Result}"); } +// ======================= +// === LIST CONTENTS ===== +// ======================= +if (listContents) +{ + Console.WriteLine("\n====================================\n"); + Console.WriteLine("Listing contents of memory"); + + var results = await memory.ListAsync(); + foreach (var result in results.Results) + { + Console.WriteLine($" - {result.SourceName} - {result.Link}"); + foreach (var partition in result.Partitions) + { + Console.WriteLine($" - {partition.Text.Substring(0, 30)}... [{result.Partitions.First().LastUpdate:D}]"); + } + } +} + // ======================= // === PURGE ============= // =======================