Add support for listing memory contents (issue 69)

dotnet/CoreLib/Memory.cs

@@ -204,4 +204,21 @@ public Task<MemoryAnswer> AskAsync(
         index = IndexExtensions.CleanName(index);
         return this._searchClient.AskAsync(index: index, question: question, filters: filters, cancellationToken: cancellationToken);
     }
+
+    public Task<SearchResult> ListAsync(
+        string? index = null,
+        MemoryFilter? filter = null,
+        ICollection<MemoryFilter>? filters = null,
+        CancellationToken cancellationToken = default)
+    {
+        if (filter != null)
+        {
+            if (filters == null) { filters = new List<MemoryFilter>(); }
+
+            filters.Add(filter);
+        }
+
+        index = IndexExtensions.CleanName(index);
+        return this._searchClient.ListAsync(index: index, filters: filters, cancellationToken: cancellationToken);
+    }
 }

dotnet/CoreLib/Search/SearchClient.cs

@@ -288,6 +288,76 @@ public async Task<MemoryAnswer> AskAsync(
         return answer;
     }
 
+    public async Task<SearchResult> ListAsync(
+        string index,
+        ICollection<MemoryFilter>? filters = null,
+        CancellationToken cancellationToken = default)
+    {
+        var result = new SearchResult
+        {
+            Query = string.Empty,
+            Results = new List<Citation>()
+        };
+
+        this._log.LogTrace("Fetching all memories");
+        IAsyncEnumerable<MemoryRecord> matches = this._vectorDb.GetListAsync(
+            indexName: index, limit: -1, filters: filters, cancellationToken: cancellationToken);
+
+        // Memories are sorted by relevance, starting from the most relevant
+        await foreach (MemoryRecord memory in matches.WithCancellation(cancellationToken))
+        {
+            // Note: a document can be composed by multiple files
+            string documentId = memory.Tags[Constants.ReservedDocumentIdTag].FirstOrDefault() ?? string.Empty;
+
+            // Identify the file in case there are multiple files
+            string fileId = memory.Tags[Constants.ReservedFileIdTag].FirstOrDefault() ?? string.Empty;
+
+            // TODO: URL to access the file
+            string linkToFile = $"{documentId}/{fileId}";
+
+            string fileContentType = memory.Tags[Constants.ReservedFileTypeTag].FirstOrDefault() ?? string.Empty;
+            string fileName = memory.Payload[Constants.ReservedPayloadFileNameField].ToString() ?? string.Empty;
+
+            var partitionText = memory.Payload[Constants.ReservedPayloadTextField].ToString()?.Trim() ?? "";
+            if (string.IsNullOrEmpty(partitionText))
+            {
+                this._log.LogError("The document partition is empty, doc: {0}", memory.Id);
+                continue;
+            }
+
+            // If the file is already in the list of citations, only add the partition
+            var citation = result.Results.FirstOrDefault(x => x.Link == linkToFile);
+            if (citation == null)
+            {
+                citation = new Citation();
+                result.Results.Add(citation);
+            }
+
+            // Add the partition to the list of citations
+            citation.Link = linkToFile;
+            citation.SourceContentType = fileContentType;
+            citation.SourceName = fileName;
+            citation.Tags = memory.Tags;
+
+#pragma warning disable CA1806 // it's ok if parsing fails
+            DateTimeOffset.TryParse(memory.Payload[Constants.ReservedPayloadLastUpdateField].ToString(), out var lastUpdate);
+#pragma warning restore CA1806
+
+            citation.Partitions.Add(new Citation.Partition
+            {
+                Text = partitionText,
+                LastUpdate = lastUpdate,
+            });
+        }
+
+        if (result.Results.Count == 0)
+        {
+            this._log.LogDebug("No memories found");
+        }
+
+        return result;
+    }
+
     private async Task<Embedding> GenerateEmbeddingAsync(string text)
     {
         this._log.LogTrace("Generating embedding for the query");

examples/001-dotnet-Serverless/Program.cs

@@ -23,6 +23,7 @@
 bool useImages = false; // Enable Azure Form Recognizer OCR to use this
 bool retrieval = true;
 bool purge = true;
+bool listContents = true;
 
 // =======================
 // === INGESTION =========
@@ -189,6 +190,25 @@ await memory.ImportWebPageAsync("https://raw.githubusercontent.com/microsoft/sem
     Console.WriteLine($"\nNews: {answer.Result}");
 }
 
+// =======================
+// === LIST CONTENTS =====
+// =======================
+if (listContents)
+{
+    Console.WriteLine("\n====================================\n");
+    Console.WriteLine("Listing contents of memory");
+
+    var results = await memory.ListAsync();
+    foreach (var result in results.Results)
+    {
+        Console.WriteLine($"  - {result.SourceName}  - {result.Link}");
+        foreach (var partition in result.Partitions)
+        {
+            Console.WriteLine($"    - {partition.Text.Substring(0, 30)}... [{result.Partitions.First().LastUpdate:D}]");
+        }
+    }
+}
+
 // =======================
 // === PURGE =============
 // =======================