From ce898e1a97daa70fa88762e530331f9e21cbbe5d Mon Sep 17 00:00:00 2001 From: akatz Date: Wed, 28 Dec 2022 17:58:36 +0100 Subject: [PATCH 01/37] Infrastructure for log writing --- ifrs17/DataModel/DataStructure.ipynb | 141 ++++++++++++++++++++++++++- 1 file changed, 137 insertions(+), 4 deletions(-) diff --git a/ifrs17/DataModel/DataStructure.ipynb b/ifrs17/DataModel/DataStructure.ipynb index a9275444..338e9882 100644 --- a/ifrs17/DataModel/DataStructure.ipynb +++ b/ifrs17/DataModel/DataStructure.ipynb @@ -59,14 +59,15 @@ "\n#r \"nuget:Systemorph.SharePoint,1.6.1\"", "\n#r \"nuget:Systemorph.OneDrive,1.6.1\"", "\n#r \"nuget:Systemorph.Scopes,1.6.1\"", - "\n#r \"nuget:Systemorph.Import,1.6.2\"", + "\n#r \"nuget:Systemorph.Import,1.6.3-ci-20221222-163048\"", "\n#r \"nuget:Systemorph.Test,1.6.1\"", - "\n#r \"nuget:Systemorph.Export,1.6.2\"", - "\n#r \"nuget:Systemorph.DataSetReader,1.6.2\"", + "\n#r \"nuget:Systemorph.Export,1.6.3-ci-20221222-163048\"", + "\n#r \"nuget:Systemorph.DataSetReader,1.6.3--ci-20221222-163048\"", "\n#r \"nuget:Systemorph.DataSource,1.6.0\"", "\n#r \"nuget:Systemorph.DataSource.Conversions,1.6.0\"", "\n#r \"nuget:Systemorph.Reporting,1.6.1\"", - "\n#r \"nuget:Systemorph.Charting,1.6.1\"" + "\n#r \"nuget:Systemorph.Charting,1.6.1\"", + "\n#r \"nuget:Systemorph.SchemaMigrations,1.6.1\"" ], "metadata": {}, "execution_count": 0, @@ -2154,6 +2155,138 @@ "metadata": {}, "execution_count": 0, "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "# Variable for Recording the Logs" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "The following variables are defined in order to keep the file information and keep trace of the logs, each time we access the DB." + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Variable that Tracks the DB Activity" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "public record KeyedMessage : KeyedRecord", + "\n{", + "\n public string Message {get; init;}", + "\n", + "\n public Guid ActivityId {get; init;}", + "\n", + "\n public DateTime MessageTime {get; init;}", + "\n", + "\n public KeyedMessage(){Id = Guid.NewGuid();}", + "\n}", + "\n", + "\npublic record Error : KeyedMessage {}", + "\n", + "\npublic record Warning : KeyedMessage {}", + "\n", + "\npublic record Info : KeyedMessage {}", + "\n" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "public record KeyedActivity : KeyedRecord", + "\n{", + "\n public string UserName {get; init;}", + "\n", + "\n public DateTime StartDateTime {get; init;}", + "\n", + "\n public DateTime EndDateTime {get; init;}", + "\n", + "\n public ActivityLogStatus Status {get; init;}", + "\n", + "\n public string Category {get; init;}", + "\n", + "\n public string Format {get; init;}", + "\n", + "\n public string VariableType {get; init;}", + "\n", + "\n public Guid ExternalFileId {get; init;}", + "\n", + "\n public KeyedActivity()", + "\n {", + "\n Id = Guid.NewGuid();", + "\n }", + "\n}" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## File Tracking Variable" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "public record KeyedFile : KeyedRecord", + "\n{", + "\n public string Name {get; init;}", + "\n", + "\n public DateTime CreationTime {get; init;}", + "\n", + "\n public byte[] SerializedFile {get; init;}", + "\n", + "\n public string Directory {get; init;}", + "\n", + "\n public long FileLength {get; init;}", + "\n ", + "\n public string ContentType {get; init;}", + "\n", + "\n public string Partition {get; init;}", + "\n", + "\n public string Source {get; init;}", + "\n", + "\n public KeyedFile()", + "\n {", + "\n Id = Guid.NewGuid();", + "\n }", + "\n}" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] } ] } \ No newline at end of file From a27ec58cde7688851487c3eca922e437b32cd90b Mon Sep 17 00:00:00 2001 From: akatz Date: Thu, 29 Dec 2022 12:18:52 +0100 Subject: [PATCH 02/37] Finally a scaffoldable model --- ifrs17/DataModel/DataStructure.ipynb | 47 +++++++--------------------- 1 file changed, 12 insertions(+), 35 deletions(-) diff --git a/ifrs17/DataModel/DataStructure.ipynb b/ifrs17/DataModel/DataStructure.ipynb index 338e9882..7c9425e7 100644 --- a/ifrs17/DataModel/DataStructure.ipynb +++ b/ifrs17/DataModel/DataStructure.ipynb @@ -2183,31 +2183,6 @@ "execution_count": 0, "outputs": [] }, - { - "cell_type": "code", - "source": [ - "public record KeyedMessage : KeyedRecord", - "\n{", - "\n public string Message {get; init;}", - "\n", - "\n public Guid ActivityId {get; init;}", - "\n", - "\n public DateTime MessageTime {get; init;}", - "\n", - "\n public KeyedMessage(){Id = Guid.NewGuid();}", - "\n}", - "\n", - "\npublic record Error : KeyedMessage {}", - "\n", - "\npublic record Warning : KeyedMessage {}", - "\n", - "\npublic record Info : KeyedMessage {}", - "\n" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, { "cell_type": "code", "source": [ @@ -2225,14 +2200,21 @@ "\n", "\n public string Format {get; init;}", "\n", + "\n [Conversion(typeof(JsonConverter))]", + "\n public string[] ErrorMessages {get; init;}", + "\n", + "\n [Conversion(typeof(JsonConverter))]", + "\n public string[] WarningMessages {get; init;}", + "\n", + "\n [Conversion(typeof(JsonConverter))]", + "\n public string[] InfoMessages {get; init;}", + "\n", "\n public string VariableType {get; init;}", "\n", - "\n public Guid ExternalFileId {get; init;}", + "\n public string FileName {get; init;}", + "\n", + "\n public string FileId {get; init;}", "\n", - "\n public KeyedActivity()", - "\n {", - "\n Id = Guid.NewGuid();", - "\n }", "\n}" ], "metadata": {}, @@ -2268,11 +2250,6 @@ "\n public string Partition {get; init;}", "\n", "\n public string Source {get; init;}", - "\n", - "\n public KeyedFile()", - "\n {", - "\n Id = Guid.NewGuid();", - "\n }", "\n}" ], "metadata": {}, From 7e0b208f79b46c8ce7ad83aca43ed93f486f17bb Mon Sep 17 00:00:00 2001 From: akatz Date: Thu, 29 Dec 2022 16:28:55 +0100 Subject: [PATCH 03/37] Moved ExecuteWithStore into the CalculationEngine importers --- ifrs17/Import/Importers.ipynb | 86 +++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index 01482d48..c4a8cb99 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -287,6 +287,92 @@ "execution_count": 0, "outputs": [] }, + { + "cell_type": "markdown", + "source": [ + "## Import and Execution with Writing Down Logs into the DB" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "using System.IO;", + "\nusing Systemorph.Vertex.DataSetReader.Csv;", + "\nusing Systemorph.Vertex.Session;", + "\nusing Systemorph.Vertex.Import.Builders;", + "\nusing Systemorph.Vertex.Import.Mappings;", + "\n", + "\npublic record StreamWrapper(Stream Stream, bool WillBeReused);", + "\n", + "\npublic static string ProcessNotification(this object obj) => obj is ActivityMessageNotification amn ? amn.Message : \"\"; ", + "\n", + "\npublic static async Task ExecuteWithStoreActivityAsync(this ImportOptionsBuilder builder, ", + "\n ISessionVariable session, ", + "\n IDataSource dataSource)", + "\n{", + "\n var log = await builder.ExecuteAsync();", + "\n var options = builder.GetImportOptions();", + "\n string formatFromFile;", + "\n byte[] content;", + "\n StreamWrapper stream = options switch", + "\n {", + "\n FileImportOptions fio => new StreamWrapper(await fio.Storage.ReadAsync(fio.FileName, session.CancellationToken), true),", + "\n StreamImportOptions streamImportOptions => new StreamWrapper(streamImportOptions.Stream, false),", + "\n StringImportOptions stringImportOptions => new StreamWrapper(new MemoryStream(Encoding.ASCII.GetBytes(stringImportOptions.Content)), true),", + "\n DataSetImportOptions dataSetImportOptions => new StreamWrapper(new MemoryStream(Encoding.ASCII.GetBytes(DataSetCsvSerializer.Serialize(dataSetImportOptions.DataSet))), true),", + "\n _ => null", + "\n };", + "\n using (MemoryStream ms = new MemoryStream())", + "\n {", + "\n await stream.Stream.CopyToAsync(ms);", + "\n content = ms.ToArray();", + "\n ms.Position = 0;", + "\n var dsRes = await builder.GetDataSetImportVariable().ReadFromStream(ms).ExecuteAsync();", + "\n formatFromFile = dsRes.Format;", + "\n if (stream.WillBeReused)", + "\n stream.Stream.Position = 0;", + "\n else", + "\n {", + "\n stream.Stream.Close();", + "\n await stream.Stream.DisposeAsync();", + "\n }", + "\n }", + "\n string fileName = options is FileImportOptions fileImportOptions ? fileImportOptions.FileName : null; ", + "\n var format = formatFromFile ?? options.Format;", + "\n var keyedFile = new KeyedFile() with { Id = Guid.NewGuid(),", + "\n SerializedFile = content, ", + "\n Name = Path.GetFileName(fileName), ", + "\n CreationTime = DateTime.UtcNow,", + "\n Directory = Path.GetDirectoryName(fileName), ", + "\n ContentType = Path.GetExtension(fileName), ", + "\n FileLength = content.Length,", + "\n Partition = null, // Andrey Katz: Options.TargetDataSource.Partion.GetCurrent(?? What do we put here, different classes might posess various partitions, e.g. Yield Curve has none ??)", + "\n Source = options.Storage.GetType().Name};", + "\n var activity = new KeyedActivity() with { Id = Guid.NewGuid(),", + "\n UserName = session.User.Name, ", + "\n StartDateTime = log.StartDateTime, ", + "\n EndDateTime = log.FinishDateTime, ", + "\n Status = log.Status, ", + "\n ErrorMessages = log.Errors.Select(x => x.ProcessNotification()).ToArray(), ", + "\n WarningMessages = log.Warnings.Select(x => x.ProcessNotification()).ToArray(), ", + "\n InfoMessages = log.Infos.Select(x => x.ProcessNotification()).ToArray(), ", + "\n Format = format,", + "\n Category = \"Import\",", + "\n FileName = keyedFile.Name, ", + "\n FileId = keyedFile.Id.ToString()};", + "\n await dataSource.UpdateAsync(keyedFile.RepeatOnce());", + "\n await dataSource.UpdateAsync(activity.RepeatOnce());", + "\n await dataSource.CommitAsync(); ", + "\n return log;", + "\n}" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, { "cell_type": "markdown", "source": [ From ef0fba33c325bc08a8fca70330d5ea126d820a21 Mon Sep 17 00:00:00 2001 From: akatz Date: Tue, 3 Jan 2023 17:59:07 +0100 Subject: [PATCH 04/37] Move the classes and the routines to a separate notebook. Some minor fixes to address the PR comments --- ifrs17/DataModel/DataStructure.ipynb | 106 +------------ ifrs17/Import/Importers.ipynb | 5 +- ifrs17/Utils/ActivityLogsAndImports.ipynb | 178 ++++++++++++++++++++++ 3 files changed, 185 insertions(+), 104 deletions(-) create mode 100644 ifrs17/Utils/ActivityLogsAndImports.ipynb diff --git a/ifrs17/DataModel/DataStructure.ipynb b/ifrs17/DataModel/DataStructure.ipynb index 7c9425e7..c7cf2366 100644 --- a/ifrs17/DataModel/DataStructure.ipynb +++ b/ifrs17/DataModel/DataStructure.ipynb @@ -59,10 +59,10 @@ "\n#r \"nuget:Systemorph.SharePoint,1.6.1\"", "\n#r \"nuget:Systemorph.OneDrive,1.6.1\"", "\n#r \"nuget:Systemorph.Scopes,1.6.1\"", - "\n#r \"nuget:Systemorph.Import,1.6.3-ci-20221222-163048\"", + "\n#r \"nuget:Systemorph.Import,1.6.3-ci-20230103-150150\"", "\n#r \"nuget:Systemorph.Test,1.6.1\"", - "\n#r \"nuget:Systemorph.Export,1.6.3-ci-20221222-163048\"", - "\n#r \"nuget:Systemorph.DataSetReader,1.6.3--ci-20221222-163048\"", + "\n#r \"nuget:Systemorph.Export,1.6.3-ci-20230103-150150\"", + "\n#r \"nuget:Systemorph.DataSetReader,1.6.3-ci-20230103-150150\"", "\n#r \"nuget:Systemorph.DataSource,1.6.0\"", "\n#r \"nuget:Systemorph.DataSource.Conversions,1.6.0\"", "\n#r \"nuget:Systemorph.Reporting,1.6.1\"", @@ -2156,106 +2156,6 @@ "execution_count": 0, "outputs": [] }, - { - "cell_type": "markdown", - "source": [ - "# Variable for Recording the Logs" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "The following variables are defined in order to keep the file information and keep trace of the logs, each time we access the DB." - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Variable that Tracks the DB Activity" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "public record KeyedActivity : KeyedRecord", - "\n{", - "\n public string UserName {get; init;}", - "\n", - "\n public DateTime StartDateTime {get; init;}", - "\n", - "\n public DateTime EndDateTime {get; init;}", - "\n", - "\n public ActivityLogStatus Status {get; init;}", - "\n", - "\n public string Category {get; init;}", - "\n", - "\n public string Format {get; init;}", - "\n", - "\n [Conversion(typeof(JsonConverter))]", - "\n public string[] ErrorMessages {get; init;}", - "\n", - "\n [Conversion(typeof(JsonConverter))]", - "\n public string[] WarningMessages {get; init;}", - "\n", - "\n [Conversion(typeof(JsonConverter))]", - "\n public string[] InfoMessages {get; init;}", - "\n", - "\n public string VariableType {get; init;}", - "\n", - "\n public string FileName {get; init;}", - "\n", - "\n public string FileId {get; init;}", - "\n", - "\n}" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## File Tracking Variable" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "public record KeyedFile : KeyedRecord", - "\n{", - "\n public string Name {get; init;}", - "\n", - "\n public DateTime CreationTime {get; init;}", - "\n", - "\n public byte[] SerializedFile {get; init;}", - "\n", - "\n public string Directory {get; init;}", - "\n", - "\n public long FileLength {get; init;}", - "\n ", - "\n public string ContentType {get; init;}", - "\n", - "\n public string Partition {get; init;}", - "\n", - "\n public string Source {get; init;}", - "\n}" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, { "cell_type": "code", "source": [ diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index c4a8cb99..807be723 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -304,6 +304,7 @@ "\nusing Systemorph.Vertex.Session;", "\nusing Systemorph.Vertex.Import.Builders;", "\nusing Systemorph.Vertex.Import.Mappings;", + "\nusing Systemorph.Vertex.DataSetReader;", "\n", "\npublic record StreamWrapper(Stream Stream, bool WillBeReused);", "\n", @@ -330,7 +331,9 @@ "\n await stream.Stream.CopyToAsync(ms);", "\n content = ms.ToArray();", "\n ms.Position = 0;", - "\n var dsRes = await builder.GetDataSetImportVariable().ReadFromStream(ms).ExecuteAsync();", + "\n DataSetImportVariable importVariable = new DataSetImportVariable(session, sess => sess.CancellationToken);", + "\n var dsRes = await importVariable.ReadFromStream(ms).ExecuteAsync();", + "\n //var dsRes = await builder.GetDataSetImportVariable().ReadFromStream(ms).ExecuteAsync();", "\n formatFromFile = dsRes.Format;", "\n if (stream.WillBeReused)", "\n stream.Stream.Position = 0;", diff --git a/ifrs17/Utils/ActivityLogsAndImports.ipynb b/ifrs17/Utils/ActivityLogsAndImports.ipynb new file mode 100644 index 00000000..89988f57 --- /dev/null +++ b/ifrs17/Utils/ActivityLogsAndImports.ipynb @@ -0,0 +1,178 @@ +{ + "metadata": { + "authors": [], + "kernelspec": { + "display_name": "Formula Framework", + "language": "C#", + "name": "C#" + }, + "language_info": { + "file_extension": ".cs", + "mimetype": "text/plain", + "name": "C#" + } + }, + "nbformat": 4, + "nbformat_minor": 5, + "cells": [ + { + "cell_type": "code", + "source": [ + "#!import \"../DataModel/DataStructure\"" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "public record KeyedActivity : KeyedRecord", + "\n{", + "\n public string Username {get; init;}", + "\n", + "\n public DateTime StartDateTime {get; init;}", + "\n", + "\n public DateTime EndDateTime {get; init;}", + "\n", + "\n public ActivityLogStatus Status {get; init;}", + "\n", + "\n public string Category {get; init;}", + "\n", + "\n public string Format {get; init;}", + "\n", + "\n [Conversion(typeof(JsonConverter))]", + "\n public string[] ErrorMessages {get; init;}", + "\n", + "\n [Conversion(typeof(JsonConverter))]", + "\n public string[] WarningMessages {get; init;}", + "\n", + "\n [Conversion(typeof(JsonConverter))]", + "\n public string[] InfoMessages {get; init;}", + "\n", + "\n public string VariableType {get; init;}", + "\n", + "\n public string FileId {get; init;}", + "\n", + "\n}" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "public record KeyedFile : KeyedRecord", + "\n{", + "\n public string Name {get; init;}", + "\n", + "\n public DateTime CreationTime {get; init;}", + "\n", + "\n public byte[] SerializedFile {get; init;}", + "\n", + "\n public string Directory {get; init;}", + "\n", + "\n public long FileLength {get; init;}", + "\n ", + "\n public string ContentType {get; init;}", + "\n", + "\n public string Partition {get; init;}", + "\n", + "\n public string Source {get; init;}", + "\n}" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "using System.IO;", + "\nusing System.Text;", + "\nusing Systemorph.Vertex.DataSetReader.Csv;", + "\nusing Systemorph.Vertex.Session;", + "\nusing Systemorph.Vertex.Import.Builders;", + "\nusing Systemorph.Vertex.Import.Mappings;", + "\nusing Systemorph.Vertex.DataSetReader;", + "\n", + "\npublic record StreamWrapper(Stream Stream, bool WillBeReused);", + "\n", + "\npublic static string ProcessNotification(this object obj) => obj is ActivityMessageNotification amn ? amn.Message : \"\"; ", + "\n", + "\npublic static async Task ExecuteWithStoreActivityAsync(this ImportOptionsBuilder builder, ", + "\n ISessionVariable session, ", + "\n IDataSource dataSource)", + "\n{", + "\n var log = await builder.ExecuteAsync();", + "\n var options = builder.GetImportOptions();", + "\n string formatFromFile;", + "\n byte[] content;", + "\n StreamWrapper stream = options switch", + "\n {", + "\n FileImportOptions fio => new StreamWrapper(await fio.Storage.ReadAsync(fio.FileName, session.CancellationToken), true),", + "\n StreamImportOptions streamImportOptions => new StreamWrapper(streamImportOptions.Stream, false),", + "\n StringImportOptions stringImportOptions => new StreamWrapper(new MemoryStream(Encoding.ASCII.GetBytes(stringImportOptions.Content)), true),", + "\n DataSetImportOptions dataSetImportOptions => new StreamWrapper(new MemoryStream(Encoding.ASCII.GetBytes(DataSetCsvSerializer.Serialize(dataSetImportOptions.DataSet))), true),", + "\n _ => null", + "\n };", + "\n using (MemoryStream ms = new MemoryStream())", + "\n {", + "\n await stream.Stream.CopyToAsync(ms);", + "\n content = ms.ToArray();", + "\n ms.Position = 0;", + "\n DataSetImportVariable importVariable = new DataSetImportVariable(session, sess => sess.CancellationToken);", + "\n var dsRes = await importVariable.ReadFromStream(ms).ExecuteAsync();", + "\n formatFromFile = dsRes.Format;", + "\n if (stream.WillBeReused)", + "\n stream.Stream.Position = 0;", + "\n else", + "\n {", + "\n stream.Stream.Close();", + "\n await stream.Stream.DisposeAsync();", + "\n }", + "\n }", + "\n string fileName = options is FileImportOptions fileImportOptions ? fileImportOptions.FileName : null; ", + "\n var format = formatFromFile ?? options.Format;", + "\n var keyedFile = new KeyedFile() with { Id = Guid.NewGuid(),", + "\n SerializedFile = content, ", + "\n Name = Path.GetFileName(fileName), ", + "\n CreationTime = DateTime.UtcNow,", + "\n Directory = Path.GetDirectoryName(fileName), ", + "\n ContentType = Path.GetExtension(fileName), ", + "\n FileLength = content.Length,", + "\n Partition = null, // Andrey Katz: Options.TargetDataSource.Partion.GetCurrent(?? What do we put here, different classes might posess various partitions, e.g. Yield Curve has none ??)", + "\n Source = options.Storage.GetType().Name};", + "\n var activity = new KeyedActivity() with { Id = Guid.NewGuid(),", + "\n Username = session.User.Name, ", + "\n StartDateTime = log.StartDateTime, ", + "\n EndDateTime = log.FinishDateTime, ", + "\n Status = log.Status, ", + "\n ErrorMessages = log.Errors.Select(x => x.ProcessNotification()).ToArray(), ", + "\n WarningMessages = log.Warnings.Select(x => x.ProcessNotification()).ToArray(), ", + "\n InfoMessages = log.Infos.Select(x => x.ProcessNotification()).ToArray(), ", + "\n Format = format,", + "\n Category = \"Import\",", + "\n FileId = keyedFile.Id.ToString()};", + "\n await dataSource.UpdateAsync(keyedFile.RepeatOnce());", + "\n await dataSource.UpdateAsync(activity.RepeatOnce());", + "\n await dataSource.CommitAsync(); ", + "\n return log;", + "\n}" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + } + ] +} \ No newline at end of file From fb6b5ad25d1fdd51aefd07a1292d15bae05e45c5 Mon Sep 17 00:00:00 2001 From: akatz Date: Tue, 3 Jan 2023 18:03:57 +0100 Subject: [PATCH 05/37] Remove obsolete parts from the importers notebook --- ifrs17/Import/Importers.ipynb | 89 ----------------------------------- 1 file changed, 89 deletions(-) diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index 807be723..01482d48 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -287,95 +287,6 @@ "execution_count": 0, "outputs": [] }, - { - "cell_type": "markdown", - "source": [ - "## Import and Execution with Writing Down Logs into the DB" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "using System.IO;", - "\nusing Systemorph.Vertex.DataSetReader.Csv;", - "\nusing Systemorph.Vertex.Session;", - "\nusing Systemorph.Vertex.Import.Builders;", - "\nusing Systemorph.Vertex.Import.Mappings;", - "\nusing Systemorph.Vertex.DataSetReader;", - "\n", - "\npublic record StreamWrapper(Stream Stream, bool WillBeReused);", - "\n", - "\npublic static string ProcessNotification(this object obj) => obj is ActivityMessageNotification amn ? amn.Message : \"\"; ", - "\n", - "\npublic static async Task ExecuteWithStoreActivityAsync(this ImportOptionsBuilder builder, ", - "\n ISessionVariable session, ", - "\n IDataSource dataSource)", - "\n{", - "\n var log = await builder.ExecuteAsync();", - "\n var options = builder.GetImportOptions();", - "\n string formatFromFile;", - "\n byte[] content;", - "\n StreamWrapper stream = options switch", - "\n {", - "\n FileImportOptions fio => new StreamWrapper(await fio.Storage.ReadAsync(fio.FileName, session.CancellationToken), true),", - "\n StreamImportOptions streamImportOptions => new StreamWrapper(streamImportOptions.Stream, false),", - "\n StringImportOptions stringImportOptions => new StreamWrapper(new MemoryStream(Encoding.ASCII.GetBytes(stringImportOptions.Content)), true),", - "\n DataSetImportOptions dataSetImportOptions => new StreamWrapper(new MemoryStream(Encoding.ASCII.GetBytes(DataSetCsvSerializer.Serialize(dataSetImportOptions.DataSet))), true),", - "\n _ => null", - "\n };", - "\n using (MemoryStream ms = new MemoryStream())", - "\n {", - "\n await stream.Stream.CopyToAsync(ms);", - "\n content = ms.ToArray();", - "\n ms.Position = 0;", - "\n DataSetImportVariable importVariable = new DataSetImportVariable(session, sess => sess.CancellationToken);", - "\n var dsRes = await importVariable.ReadFromStream(ms).ExecuteAsync();", - "\n //var dsRes = await builder.GetDataSetImportVariable().ReadFromStream(ms).ExecuteAsync();", - "\n formatFromFile = dsRes.Format;", - "\n if (stream.WillBeReused)", - "\n stream.Stream.Position = 0;", - "\n else", - "\n {", - "\n stream.Stream.Close();", - "\n await stream.Stream.DisposeAsync();", - "\n }", - "\n }", - "\n string fileName = options is FileImportOptions fileImportOptions ? fileImportOptions.FileName : null; ", - "\n var format = formatFromFile ?? options.Format;", - "\n var keyedFile = new KeyedFile() with { Id = Guid.NewGuid(),", - "\n SerializedFile = content, ", - "\n Name = Path.GetFileName(fileName), ", - "\n CreationTime = DateTime.UtcNow,", - "\n Directory = Path.GetDirectoryName(fileName), ", - "\n ContentType = Path.GetExtension(fileName), ", - "\n FileLength = content.Length,", - "\n Partition = null, // Andrey Katz: Options.TargetDataSource.Partion.GetCurrent(?? What do we put here, different classes might posess various partitions, e.g. Yield Curve has none ??)", - "\n Source = options.Storage.GetType().Name};", - "\n var activity = new KeyedActivity() with { Id = Guid.NewGuid(),", - "\n UserName = session.User.Name, ", - "\n StartDateTime = log.StartDateTime, ", - "\n EndDateTime = log.FinishDateTime, ", - "\n Status = log.Status, ", - "\n ErrorMessages = log.Errors.Select(x => x.ProcessNotification()).ToArray(), ", - "\n WarningMessages = log.Warnings.Select(x => x.ProcessNotification()).ToArray(), ", - "\n InfoMessages = log.Infos.Select(x => x.ProcessNotification()).ToArray(), ", - "\n Format = format,", - "\n Category = \"Import\",", - "\n FileName = keyedFile.Name, ", - "\n FileId = keyedFile.Id.ToString()};", - "\n await dataSource.UpdateAsync(keyedFile.RepeatOnce());", - "\n await dataSource.UpdateAsync(activity.RepeatOnce());", - "\n await dataSource.CommitAsync(); ", - "\n return log;", - "\n}" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, { "cell_type": "markdown", "source": [ From 8098443b0250df44c9d3224b0284a4eb71863311 Mon Sep 17 00:00:00 2001 From: akatz Date: Tue, 3 Jan 2023 18:45:20 +0100 Subject: [PATCH 06/37] Thoughts on API --- ifrs17/Utils/ActivityLogsAndImports.ipynb | 30 +++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/ifrs17/Utils/ActivityLogsAndImports.ipynb b/ifrs17/Utils/ActivityLogsAndImports.ipynb index 89988f57..e40d1227 100644 --- a/ifrs17/Utils/ActivityLogsAndImports.ipynb +++ b/ifrs17/Utils/ActivityLogsAndImports.ipynb @@ -165,6 +165,36 @@ "execution_count": 0, "outputs": [] }, + { + "cell_type": "markdown", + "source": [ + "Thoughts about API. First: what do we save if the import is not from the File, but rather stream, string, or data set? Suggestions:", + "\n1. Need one more abstraction level: KeyedImport", + "\n2. KeyedImport should include: Guid Id, Type (string, stream, data set or file), creation time and serialized content and content length. The latter has nothing to do with the file per se, it is simply a byte length of the import. These are generic types.", + "\n3. KeyedFile should directly inherit from the Keyed Import, and this can include on top of that Directory, content type, name.", + "\n4. FileId in the KeyedActivity should probably be ImportId, as the import is not necesserily from a file. ", + "\n5. Name might be either be a file-only attribute, or be assigned autimatically if not given. In this case it might be an identity property.", + "\n ", + "\nQuestion: how do we accomodate this in our tables? We can have separate tables for strings, data sets and streams. If in U+ some of them will stay empty, this does not mean that they will not be populated in other projects. In fact, the type of import can be moved to the KeyedActivity." + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "On the API side. Certain actions must be moved to the KeyedImport and maybe even made private (in fact, they should not exposed to the outside user):", + "\n1. Determination of content and length. ", + "\n2. Creation date and time", + "\n3. Type of input. ", + "\n ", + "\nPrposed API: Initialize the import via options -> determine the content, time, and type, which are called automatically from constructor. " + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, { "cell_type": "code", "source": [ From b0bb67b1eb27898d906da9dbdb1d8c5ab8bb14ba Mon Sep 17 00:00:00 2001 From: akatz Date: Wed, 4 Jan 2023 18:16:51 +0100 Subject: [PATCH 07/37] Reshaped, allowed for writeup for any type of import. --- ifrs17/Utils/ActivityLogsAndImports.ipynb | 280 +++++++++++++++------- 1 file changed, 187 insertions(+), 93 deletions(-) diff --git a/ifrs17/Utils/ActivityLogsAndImports.ipynb b/ifrs17/Utils/ActivityLogsAndImports.ipynb index e40d1227..6f0c1f18 100644 --- a/ifrs17/Utils/ActivityLogsAndImports.ipynb +++ b/ifrs17/Utils/ActivityLogsAndImports.ipynb @@ -24,6 +24,25 @@ "execution_count": 0, "outputs": [] }, + { + "cell_type": "code", + "source": [ + "using System.IO;", + "\nusing System.Text;", + "\nusing Systemorph.Vertex.DataSetReader.Csv;", + "\nusing Systemorph.Vertex.Session;", + "\nusing Systemorph.Vertex.Import.Builders;", + "\nusing Systemorph.Vertex.Import.Mappings;", + "\nusing Systemorph.Vertex.DataSetReader;", + "\n", + "\npublic record StreamWrapper(Stream Stream, bool WillBeReused);", + "\n", + "\npublic static string ProcessNotification(this object obj) => obj is ActivityMessageNotification amn ? amn.Message : \"\"; " + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, { "cell_type": "code", "source": [ @@ -39,8 +58,6 @@ "\n", "\n public string Category {get; init;}", "\n", - "\n public string Format {get; init;}", - "\n", "\n [Conversion(typeof(JsonConverter))]", "\n public string[] ErrorMessages {get; init;}", "\n", @@ -50,9 +67,19 @@ "\n [Conversion(typeof(JsonConverter))]", "\n public string[] InfoMessages {get; init;}", "\n", - "\n public string VariableType {get; init;}", + "\n public Guid SourceId {get; init;} // Check if converting to Guid causes issues. If not, leave Guid.", "\n", - "\n public string FileId {get; init;}", + "\n public KeyedActivity(ActivityLog log, ISessionVariable session)", + "\n {", + "\n Id = Guid.NewGuid();", + "\n Username = session.User.Name;", + "\n StartDateTime = log.StartDateTime;", + "\n EndDateTime = log.FinishDateTime;", + "\n Status = log.Status;", + "\n ErrorMessages = log.Errors.Select(x => x.ProcessNotification()).Distinct().ToArray();", + "\n WarningMessages = log.Warnings.Select(x => x.ProcessNotification()).Distinct().ToArray();", + "\n InfoMessages = log.Infos.Select(x => x.ProcessNotification()).Distinct().ToArray();", + "\n }", "\n", "\n}" ], @@ -63,23 +90,100 @@ { "cell_type": "code", "source": [ - "public record KeyedFile : KeyedRecord", + "public abstract record ImportWithKey : KeyedRecord", "\n{", - "\n public string Name {get; init;}", - "\n", "\n public DateTime CreationTime {get; init;}", + "\n public byte[] SerilaizedContent {get; init;}", + "\n public uint Length {get; init;}", + "\n public string Format {get; init;}", + "\n private protected ImportOptions importOptions {get; init;}", + "\n", + "\n public async Task InitializeImportDataAsync(ISessionVariable session)", + "\n {", + "\n var stream = await GenerateStreamWrapperAsync(session);", + "\n var formatAndContent = await GetInformationFromStreamAsync(stream, session);", + "\n return this with{CreationTime = DateTime.UtcNow, ", + "\n Format = formatAndContent.Format ?? importOptions.Format,", + "\n SerilaizedContent = formatAndContent.Content,", + "\n Length = (uint)SerilaizedContent.Length,", + "\n };", + "\n }", "\n", - "\n public byte[] SerializedFile {get; init;}", + "\n private async Task GenerateStreamWrapperAsync(ISessionVariable session)", + "\n {", + "\n StreamWrapper stream = importOptions switch", + "\n {", + "\n FileImportOptions fio => new StreamWrapper(await fio.Storage.ReadAsync(fio.FileName, session.CancellationToken), true),", + "\n StreamImportOptions streamImportOptions => new StreamWrapper(streamImportOptions.Stream, false),", + "\n StringImportOptions stringImportOptions => new StreamWrapper(new MemoryStream(Encoding.ASCII.GetBytes(stringImportOptions.Content)), true),", + "\n DataSetImportOptions dataSetImportOptions => new StreamWrapper(new MemoryStream(Encoding.ASCII.GetBytes(DataSetCsvSerializer.Serialize(dataSetImportOptions.DataSet))), true),", + "\n _ => null", + "\n };", + "\n return stream;", + "\n }", "\n", - "\n public string Directory {get; init;}", + "\n private async Task<(string Format, byte[] Content)> GetInformationFromStreamAsync(StreamWrapper stream, ISessionVariable session)", + "\n {", + "\n byte[] content;", + "\n string format;", + "\n using (MemoryStream ms = new MemoryStream())", + "\n {", + "\n await stream.Stream.CopyToAsync(ms);", + "\n content = ms.ToArray();", + "\n ms.Position = 0;", + "\n DataSetImportVariable importVariable = new DataSetImportVariable(session, sess => sess.CancellationToken);", + "\n var dsRes = await importVariable.ReadFromStream(ms).ExecuteAsync();", + "\n format = dsRes.Format;", + "\n if (stream.WillBeReused)", + "\n stream.Stream.Position = 0;", + "\n else", + "\n {", + "\n stream.Stream.Close();", + "\n await stream.Stream.DisposeAsync();", + "\n }", + "\n }", + "\n return (format, content);", + "\n }", + "\n}" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "public record KeyedFile : ImportWithKey", + "\n{", + "\n public string Name {get; init;}", "\n", - "\n public long FileLength {get; init;}", + "\n public string Directory {get; init;}", "\n ", "\n public string ContentType {get; init;}", "\n", - "\n public string Partition {get; init;}", + "\n [Conversion(typeof(JsonConverter))]", + "\n public string[] Partition {get; init;}", "\n", "\n public string Source {get; init;}", + "\n", + "\n public KeyedFile(FileImportOptions options)", + "\n {", + "\n importOptions = options;", + "\n string fileName = options.FileName; ", + "\n Id = Guid.NewGuid();", + "\n Name = Path.GetFileName(fileName);", + "\n Directory = Path.GetDirectoryName(fileName); ", + "\n ContentType = Path.GetExtension(fileName);", + "\n Source = options.Storage.GetType().Name; ", + "\n Partition = GetInvolvedPartitions(options);", + "\n // Andrey Katz: Options.TargetDataSource.Partion.GetCurrent(?? What do we put here, different classes might posess various partitions, e.g. Yield Curve has none ??)", + "\n }", + "\n", + "\n private string[] GetInvolvedPartitions(ImportOptions options)", + "\n {", + "\n //Andrey Katz: Get all the relevant partitions here ", + "\n return null;", + "\n }", "\n}" ], "metadata": {}, @@ -89,76 +193,32 @@ { "cell_type": "code", "source": [ - "using System.IO;", - "\nusing System.Text;", - "\nusing Systemorph.Vertex.DataSetReader.Csv;", - "\nusing Systemorph.Vertex.Session;", - "\nusing Systemorph.Vertex.Import.Builders;", - "\nusing Systemorph.Vertex.Import.Mappings;", - "\nusing Systemorph.Vertex.DataSetReader;", - "\n", - "\npublic record StreamWrapper(Stream Stream, bool WillBeReused);", - "\n", - "\npublic static string ProcessNotification(this object obj) => obj is ActivityMessageNotification amn ? amn.Message : \"\"; ", - "\n", - "\npublic static async Task ExecuteWithStoreActivityAsync(this ImportOptionsBuilder builder, ", - "\n ISessionVariable session, ", - "\n IDataSource dataSource)", + "public record KeyedStringImport : ImportWithKey", "\n{", - "\n var log = await builder.ExecuteAsync();", - "\n var options = builder.GetImportOptions();", - "\n string formatFromFile;", - "\n byte[] content;", - "\n StreamWrapper stream = options switch", + "\n public string Content {get; init;}", + "\n", + "\n public KeyedStringImport(StringImportOptions options)", "\n {", - "\n FileImportOptions fio => new StreamWrapper(await fio.Storage.ReadAsync(fio.FileName, session.CancellationToken), true),", - "\n StreamImportOptions streamImportOptions => new StreamWrapper(streamImportOptions.Stream, false),", - "\n StringImportOptions stringImportOptions => new StreamWrapper(new MemoryStream(Encoding.ASCII.GetBytes(stringImportOptions.Content)), true),", - "\n DataSetImportOptions dataSetImportOptions => new StreamWrapper(new MemoryStream(Encoding.ASCII.GetBytes(DataSetCsvSerializer.Serialize(dataSetImportOptions.DataSet))), true),", - "\n _ => null", - "\n };", - "\n using (MemoryStream ms = new MemoryStream())", + "\n importOptions = options;", + "\n Id = Guid.NewGuid();", + "\n Content = options.Content;", + "\n }", + "\n}" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "public record KeyedDataSetImport : ImportWithKey", + "\n{", + "\n public KeyedDataSetImport(DataSetImportOptions options)", "\n {", - "\n await stream.Stream.CopyToAsync(ms);", - "\n content = ms.ToArray();", - "\n ms.Position = 0;", - "\n DataSetImportVariable importVariable = new DataSetImportVariable(session, sess => sess.CancellationToken);", - "\n var dsRes = await importVariable.ReadFromStream(ms).ExecuteAsync();", - "\n formatFromFile = dsRes.Format;", - "\n if (stream.WillBeReused)", - "\n stream.Stream.Position = 0;", - "\n else", - "\n {", - "\n stream.Stream.Close();", - "\n await stream.Stream.DisposeAsync();", - "\n }", + "\n importOptions = options;", + "\n Id = Guid.NewGuid();", "\n }", - "\n string fileName = options is FileImportOptions fileImportOptions ? fileImportOptions.FileName : null; ", - "\n var format = formatFromFile ?? options.Format;", - "\n var keyedFile = new KeyedFile() with { Id = Guid.NewGuid(),", - "\n SerializedFile = content, ", - "\n Name = Path.GetFileName(fileName), ", - "\n CreationTime = DateTime.UtcNow,", - "\n Directory = Path.GetDirectoryName(fileName), ", - "\n ContentType = Path.GetExtension(fileName), ", - "\n FileLength = content.Length,", - "\n Partition = null, // Andrey Katz: Options.TargetDataSource.Partion.GetCurrent(?? What do we put here, different classes might posess various partitions, e.g. Yield Curve has none ??)", - "\n Source = options.Storage.GetType().Name};", - "\n var activity = new KeyedActivity() with { Id = Guid.NewGuid(),", - "\n Username = session.User.Name, ", - "\n StartDateTime = log.StartDateTime, ", - "\n EndDateTime = log.FinishDateTime, ", - "\n Status = log.Status, ", - "\n ErrorMessages = log.Errors.Select(x => x.ProcessNotification()).ToArray(), ", - "\n WarningMessages = log.Warnings.Select(x => x.ProcessNotification()).ToArray(), ", - "\n InfoMessages = log.Infos.Select(x => x.ProcessNotification()).ToArray(), ", - "\n Format = format,", - "\n Category = \"Import\",", - "\n FileId = keyedFile.Id.ToString()};", - "\n await dataSource.UpdateAsync(keyedFile.RepeatOnce());", - "\n await dataSource.UpdateAsync(activity.RepeatOnce());", - "\n await dataSource.CommitAsync(); ", - "\n return log;", "\n}" ], "metadata": {}, @@ -166,30 +226,64 @@ "outputs": [] }, { - "cell_type": "markdown", + "cell_type": "code", "source": [ - "Thoughts about API. First: what do we save if the import is not from the File, but rather stream, string, or data set? Suggestions:", - "\n1. Need one more abstraction level: KeyedImport", - "\n2. KeyedImport should include: Guid Id, Type (string, stream, data set or file), creation time and serialized content and content length. The latter has nothing to do with the file per se, it is simply a byte length of the import. These are generic types.", - "\n3. KeyedFile should directly inherit from the Keyed Import, and this can include on top of that Directory, content type, name.", - "\n4. FileId in the KeyedActivity should probably be ImportId, as the import is not necesserily from a file. ", - "\n5. Name might be either be a file-only attribute, or be assigned autimatically if not given. In this case it might be an identity property.", - "\n ", - "\nQuestion: how do we accomodate this in our tables? We can have separate tables for strings, data sets and streams. If in U+ some of them will stay empty, this does not mean that they will not be populated in other projects. In fact, the type of import can be moved to the KeyedActivity." + "public record KeyedStreamImport : ImportWithKey", + "\n{", + "\n public KeyedStreamImport(StreamImportOptions options)", + "\n {", + "\n importOptions = options;", + "\n Id = Guid.NewGuid();", + "\n }", + "\n}" ], "metadata": {}, "execution_count": 0, "outputs": [] }, { - "cell_type": "markdown", + "cell_type": "code", "source": [ - "On the API side. Certain actions must be moved to the KeyedImport and maybe even made private (in fact, they should not exposed to the outside user):", - "\n1. Determination of content and length. ", - "\n2. Creation date and time", - "\n3. Type of input. ", - "\n ", - "\nPrposed API: Initialize the import via options -> determine the content, time, and type, which are called automatically from constructor. " + "public static async Task ExecuteWithStoreActivityAsync(this ImportOptionsBuilder builder, ", + "\n ISessionVariable session, ", + "\n IDataSource dataSource)", + "\n{", + "\n var log = await builder.ExecuteAsync();", + "\n var options = builder.GetImportOptions();", + "\n var activity = new KeyedActivity(log, session);", + "\n switch(options)", + "\n {", + "\n case FileImportOptions fio:", + "\n var keyedFile = await (new KeyedFile(fio)).InitializeImportDataAsync(session);", + "\n activity = activity with {SourceId = keyedFile.Id, ", + "\n Category = \"Import from File\"};", + "\n if (keyedFile is KeyedFile kf) await dataSource.UpdateAsync(kf.RepeatOnce());", + "\n break;", + "\n case StringImportOptions sgio:", + "\n var keyedStringImport = await (new KeyedStringImport(sgio)).InitializeImportDataAsync(session);", + "\n activity = activity with {SourceId = keyedStringImport.Id, ", + "\n Category = \"Import from String\"};", + "\n if (keyedStringImport is KeyedStringImport ksi) await dataSource.UpdateAsync(ksi.RepeatOnce());", + "\n break;", + "\n case StreamImportOptions smio:", + "\n var keyedStreamImport = await (new KeyedStreamImport(smio)).InitializeImportDataAsync(session);", + "\n activity = activity with {SourceId = keyedStreamImport.Id, ", + "\n Category = \"Import from Stream\"};", + "\n if (keyedStreamImport is KeyedStreamImport ksti) await dataSource.UpdateAsync(ksti.RepeatOnce());", + "\n break;", + "\n case DataSetImportOptions dsio:", + "\n var keyedDataSetImport = await (new KeyedDataSetImport(dsio)).InitializeImportDataAsync(session);", + "\n activity = activity with {SourceId = keyedDataSetImport.Id, ", + "\n Category = \"Import from Data Set\"};", + "\n if (keyedDataSetImport is KeyedDataSetImport kdsi) await dataSource.UpdateAsync(kdsi.RepeatOnce());", + "\n break;", + "\n default:", + "\n break;", + "\n } ", + "\n await dataSource.UpdateAsync(activity.RepeatOnce());", + "\n await dataSource.CommitAsync(); ", + "\n return log;", + "\n}" ], "metadata": {}, "execution_count": 0, From d50a008d867a44b2fecedebeced3894492477584 Mon Sep 17 00:00:00 2001 From: akatz Date: Thu, 5 Jan 2023 11:10:59 +0100 Subject: [PATCH 08/37] typo fixed and castings --- ifrs17/Utils/ActivityLogsAndImports.ipynb | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ifrs17/Utils/ActivityLogsAndImports.ipynb b/ifrs17/Utils/ActivityLogsAndImports.ipynb index 6f0c1f18..4e8c5ee8 100644 --- a/ifrs17/Utils/ActivityLogsAndImports.ipynb +++ b/ifrs17/Utils/ActivityLogsAndImports.ipynb @@ -93,7 +93,7 @@ "public abstract record ImportWithKey : KeyedRecord", "\n{", "\n public DateTime CreationTime {get; init;}", - "\n public byte[] SerilaizedContent {get; init;}", + "\n public byte[] SerializedContent {get; init;}", "\n public uint Length {get; init;}", "\n public string Format {get; init;}", "\n private protected ImportOptions importOptions {get; init;}", @@ -104,8 +104,8 @@ "\n var formatAndContent = await GetInformationFromStreamAsync(stream, session);", "\n return this with{CreationTime = DateTime.UtcNow, ", "\n Format = formatAndContent.Format ?? importOptions.Format,", - "\n SerilaizedContent = formatAndContent.Content,", - "\n Length = (uint)SerilaizedContent.Length,", + "\n SerializedContent = formatAndContent.Content,", + "\n Length = (uint)SerializedContent.Length,", "\n };", "\n }", "\n", @@ -181,6 +181,7 @@ "\n", "\n private string[] GetInvolvedPartitions(ImportOptions options)", "\n {", + "\n // TODO", "\n //Andrey Katz: Get all the relevant partitions here ", "\n return null;", "\n }", @@ -257,25 +258,25 @@ "\n var keyedFile = await (new KeyedFile(fio)).InitializeImportDataAsync(session);", "\n activity = activity with {SourceId = keyedFile.Id, ", "\n Category = \"Import from File\"};", - "\n if (keyedFile is KeyedFile kf) await dataSource.UpdateAsync(kf.RepeatOnce());", + "\n await dataSource.UpdateAsync((keyedFile as KeyedFile).RepeatOnce());", "\n break;", "\n case StringImportOptions sgio:", "\n var keyedStringImport = await (new KeyedStringImport(sgio)).InitializeImportDataAsync(session);", "\n activity = activity with {SourceId = keyedStringImport.Id, ", "\n Category = \"Import from String\"};", - "\n if (keyedStringImport is KeyedStringImport ksi) await dataSource.UpdateAsync(ksi.RepeatOnce());", + "\n await dataSource.UpdateAsync((keyedStringImport as KeyedStringImport).RepeatOnce());", "\n break;", "\n case StreamImportOptions smio:", "\n var keyedStreamImport = await (new KeyedStreamImport(smio)).InitializeImportDataAsync(session);", "\n activity = activity with {SourceId = keyedStreamImport.Id, ", "\n Category = \"Import from Stream\"};", - "\n if (keyedStreamImport is KeyedStreamImport ksti) await dataSource.UpdateAsync(ksti.RepeatOnce());", + "\n await dataSource.UpdateAsync((keyedStreamImport as KeyedStreamImport).RepeatOnce());", "\n break;", "\n case DataSetImportOptions dsio:", "\n var keyedDataSetImport = await (new KeyedDataSetImport(dsio)).InitializeImportDataAsync(session);", "\n activity = activity with {SourceId = keyedDataSetImport.Id, ", "\n Category = \"Import from Data Set\"};", - "\n if (keyedDataSetImport is KeyedDataSetImport kdsi) await dataSource.UpdateAsync(kdsi.RepeatOnce());", + "\n await dataSource.UpdateAsync((keyedDataSetImport as KeyedDataSetImport).RepeatOnce());", "\n break;", "\n default:", "\n break;", From 9d2a8600e15b1a539050caf2ab83f607993ae83f Mon Sep 17 00:00:00 2001 From: akatz Date: Thu, 5 Jan 2023 11:28:29 +0100 Subject: [PATCH 09/37] move all the casts where they belong --- ifrs17/Utils/ActivityLogsAndImports.ipynb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ifrs17/Utils/ActivityLogsAndImports.ipynb b/ifrs17/Utils/ActivityLogsAndImports.ipynb index 4e8c5ee8..269b8587 100644 --- a/ifrs17/Utils/ActivityLogsAndImports.ipynb +++ b/ifrs17/Utils/ActivityLogsAndImports.ipynb @@ -255,28 +255,28 @@ "\n switch(options)", "\n {", "\n case FileImportOptions fio:", - "\n var keyedFile = await (new KeyedFile(fio)).InitializeImportDataAsync(session);", + "\n var keyedFile = await (new KeyedFile(fio)).InitializeImportDataAsync(session) as KeyedFile;", "\n activity = activity with {SourceId = keyedFile.Id, ", "\n Category = \"Import from File\"};", - "\n await dataSource.UpdateAsync((keyedFile as KeyedFile).RepeatOnce());", + "\n await dataSource.UpdateAsync(keyedFile.RepeatOnce());", "\n break;", "\n case StringImportOptions sgio:", - "\n var keyedStringImport = await (new KeyedStringImport(sgio)).InitializeImportDataAsync(session);", + "\n var keyedStringImport = await (new KeyedStringImport(sgio)).InitializeImportDataAsync(session) as KeyedStringImport;", "\n activity = activity with {SourceId = keyedStringImport.Id, ", "\n Category = \"Import from String\"};", - "\n await dataSource.UpdateAsync((keyedStringImport as KeyedStringImport).RepeatOnce());", + "\n await dataSource.UpdateAsync(keyedStringImport.RepeatOnce());", "\n break;", "\n case StreamImportOptions smio:", - "\n var keyedStreamImport = await (new KeyedStreamImport(smio)).InitializeImportDataAsync(session);", + "\n var keyedStreamImport = await (new KeyedStreamImport(smio)).InitializeImportDataAsync(session) as KeyedStreamImport;", "\n activity = activity with {SourceId = keyedStreamImport.Id, ", "\n Category = \"Import from Stream\"};", - "\n await dataSource.UpdateAsync((keyedStreamImport as KeyedStreamImport).RepeatOnce());", + "\n await dataSource.UpdateAsync(keyedStreamImport.RepeatOnce());", "\n break;", "\n case DataSetImportOptions dsio:", - "\n var keyedDataSetImport = await (new KeyedDataSetImport(dsio)).InitializeImportDataAsync(session);", + "\n var keyedDataSetImport = await (new KeyedDataSetImport(dsio)).InitializeImportDataAsync(session) as KeyedDataSetImport;", "\n activity = activity with {SourceId = keyedDataSetImport.Id, ", "\n Category = \"Import from Data Set\"};", - "\n await dataSource.UpdateAsync((keyedDataSetImport as KeyedDataSetImport).RepeatOnce());", + "\n await dataSource.UpdateAsync(keyedDataSetImport.RepeatOnce());", "\n break;", "\n default:", "\n break;", From 6d266a2ce4fe294e5e64739cbc141d376d1aa8fd Mon Sep 17 00:00:00 2001 From: akatz Date: Thu, 5 Jan 2023 11:38:43 +0100 Subject: [PATCH 10/37] Throws an exception if the Import Options does not belong to the correct class --- ifrs17/Utils/ActivityLogsAndImports.ipynb | 1 + 1 file changed, 1 insertion(+) diff --git a/ifrs17/Utils/ActivityLogsAndImports.ipynb b/ifrs17/Utils/ActivityLogsAndImports.ipynb index 269b8587..476c1501 100644 --- a/ifrs17/Utils/ActivityLogsAndImports.ipynb +++ b/ifrs17/Utils/ActivityLogsAndImports.ipynb @@ -279,6 +279,7 @@ "\n await dataSource.UpdateAsync(keyedDataSetImport.RepeatOnce());", "\n break;", "\n default:", + "\n throw new Exception(\"Import Options object is not an instance of an appropriate class.\");", "\n break;", "\n } ", "\n await dataSource.UpdateAsync(activity.RepeatOnce());", From 36421cc17d8914534f170a273e7c7b1220a33031 Mon Sep 17 00:00:00 2001 From: akatz Date: Thu, 5 Jan 2023 14:18:20 +0100 Subject: [PATCH 11/37] Add trivial constructors. Still proudly refuses to scaffold :( --- ifrs17/Utils/ActivityLogsAndImports.ipynb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ifrs17/Utils/ActivityLogsAndImports.ipynb b/ifrs17/Utils/ActivityLogsAndImports.ipynb index 476c1501..705751d0 100644 --- a/ifrs17/Utils/ActivityLogsAndImports.ipynb +++ b/ifrs17/Utils/ActivityLogsAndImports.ipynb @@ -81,6 +81,8 @@ "\n InfoMessages = log.Infos.Select(x => x.ProcessNotification()).Distinct().ToArray();", "\n }", "\n", + "\n public KeyedActivity(){}", + "\n", "\n}" ], "metadata": {}, @@ -179,6 +181,9 @@ "\n // Andrey Katz: Options.TargetDataSource.Partion.GetCurrent(?? What do we put here, different classes might posess various partitions, e.g. Yield Curve has none ??)", "\n }", "\n", + "\n public KeyedFile(){}", + "\n ", + "\n", "\n private string[] GetInvolvedPartitions(ImportOptions options)", "\n {", "\n // TODO", @@ -204,6 +209,9 @@ "\n Id = Guid.NewGuid();", "\n Content = options.Content;", "\n }", + "\n", + "\n public KeyedStringImport(){}", + "\n ", "\n}" ], "metadata": {}, @@ -220,6 +228,9 @@ "\n importOptions = options;", "\n Id = Guid.NewGuid();", "\n }", + "\n", + "\n public KeyedDataSetImport(){}", + "\n ", "\n}" ], "metadata": {}, @@ -236,6 +247,9 @@ "\n importOptions = options;", "\n Id = Guid.NewGuid();", "\n }", + "\n", + "\n public KeyedStreamImport(){}", + "\n ", "\n}" ], "metadata": {}, From c51d49773ee72bd13a159d608eee6e62c2123cc2 Mon Sep 17 00:00:00 2001 From: akatz Date: Fri, 6 Jan 2023 12:12:25 +0100 Subject: [PATCH 12/37] multiple bugs fixed. Not yet sure, if it is the final version, but getting very close there --- ifrs17/Utils/ActivityLogsAndImports.ipynb | 78 +++++++++++++++-------- 1 file changed, 53 insertions(+), 25 deletions(-) diff --git a/ifrs17/Utils/ActivityLogsAndImports.ipynb b/ifrs17/Utils/ActivityLogsAndImports.ipynb index 705751d0..12e96868 100644 --- a/ifrs17/Utils/ActivityLogsAndImports.ipynb +++ b/ifrs17/Utils/ActivityLogsAndImports.ipynb @@ -81,7 +81,10 @@ "\n InfoMessages = log.Infos.Select(x => x.ProcessNotification()).Distinct().ToArray();", "\n }", "\n", - "\n public KeyedActivity(){}", + "\n public KeyedActivity(Guid id)", + "\n {", + "\n Id = id;", + "\n }", "\n", "\n}" ], @@ -96,24 +99,25 @@ "\n{", "\n public DateTime CreationTime {get; init;}", "\n public byte[] SerializedContent {get; init;}", - "\n public uint Length {get; init;}", + "\n public uint? Length {get; init;}", "\n public string Format {get; init;}", - "\n private protected ImportOptions importOptions {get; init;}", + "\n protected ImportOptions Options {get; set;}", + "\n protected IDataSetImportVariable DataSetReader{get; set;}", "\n", "\n public async Task InitializeImportDataAsync(ISessionVariable session)", "\n {", "\n var stream = await GenerateStreamWrapperAsync(session);", "\n var formatAndContent = await GetInformationFromStreamAsync(stream, session);", "\n return this with{CreationTime = DateTime.UtcNow, ", - "\n Format = formatAndContent.Format ?? importOptions.Format,", + "\n Format = formatAndContent.Format ?? Options.Format,", "\n SerializedContent = formatAndContent.Content,", - "\n Length = (uint)SerializedContent.Length,", + "\n Length = SerializedContent != null ? (uint)SerializedContent.Length : null,", "\n };", "\n }", "\n", "\n private async Task GenerateStreamWrapperAsync(ISessionVariable session)", "\n {", - "\n StreamWrapper stream = importOptions switch", + "\n StreamWrapper stream = Options switch", "\n {", "\n FileImportOptions fio => new StreamWrapper(await fio.Storage.ReadAsync(fio.FileName, session.CancellationToken), true),", "\n StreamImportOptions streamImportOptions => new StreamWrapper(streamImportOptions.Stream, false),", @@ -133,8 +137,7 @@ "\n await stream.Stream.CopyToAsync(ms);", "\n content = ms.ToArray();", "\n ms.Position = 0;", - "\n DataSetImportVariable importVariable = new DataSetImportVariable(session, sess => sess.CancellationToken);", - "\n var dsRes = await importVariable.ReadFromStream(ms).ExecuteAsync();", + "\n var dsRes = await DataSetReader.ReadFromStream(ms).ExecuteAsync();", "\n format = dsRes.Format;", "\n if (stream.WillBeReused)", "\n stream.Stream.Position = 0;", @@ -168,9 +171,10 @@ "\n", "\n public string Source {get; init;}", "\n", - "\n public KeyedFile(FileImportOptions options)", + "\n public KeyedFile(FileImportOptions options, IDataSetImportVariable importVariable)", "\n {", - "\n importOptions = options;", + "\n Options = options;", + "\n DataSetReader = importVariable;", "\n string fileName = options.FileName; ", "\n Id = Guid.NewGuid();", "\n Name = Path.GetFileName(fileName);", @@ -181,7 +185,12 @@ "\n // Andrey Katz: Options.TargetDataSource.Partion.GetCurrent(?? What do we put here, different classes might posess various partitions, e.g. Yield Curve has none ??)", "\n }", "\n", - "\n public KeyedFile(){}", + "\n public KeyedFile(Guid id)", + "\n {", + "\n Id = id;", + "\n Options = null;", + "\n DataSetReader = null;", + "\n }", "\n ", "\n", "\n private string[] GetInvolvedPartitions(ImportOptions options)", @@ -203,14 +212,20 @@ "\n{", "\n public string Content {get; init;}", "\n", - "\n public KeyedStringImport(StringImportOptions options)", + "\n public KeyedStringImport(StringImportOptions options, IDataSetImportVariable importVariable)", "\n {", - "\n importOptions = options;", + "\n Options = options;", + "\n DataSetReader = importVariable;", "\n Id = Guid.NewGuid();", "\n Content = options.Content;", "\n }", "\n", - "\n public KeyedStringImport(){}", + "\n public KeyedStringImport(Guid id)", + "\n {", + "\n Id = id;", + "\n Options = null;", + "\n DataSetReader = null;", + "\n }", "\n ", "\n}" ], @@ -223,13 +238,19 @@ "source": [ "public record KeyedDataSetImport : ImportWithKey", "\n{", - "\n public KeyedDataSetImport(DataSetImportOptions options)", + "\n public KeyedDataSetImport(DataSetImportOptions options, IDataSetImportVariable importVariable)", "\n {", - "\n importOptions = options;", + "\n DataSetReader = importVariable;", + "\n Options = options;", "\n Id = Guid.NewGuid();", "\n }", "\n", - "\n public KeyedDataSetImport(){}", + "\n public KeyedDataSetImport(Guid id)", + "\n {", + "\n Id = id;", + "\n Options = null;", + "\n DataSetReader = null;", + "\n }", "\n ", "\n}" ], @@ -242,13 +263,19 @@ "source": [ "public record KeyedStreamImport : ImportWithKey", "\n{", - "\n public KeyedStreamImport(StreamImportOptions options)", + "\n public KeyedStreamImport(StreamImportOptions options, IDataSetImportVariable importVariable)", "\n {", - "\n importOptions = options;", + "\n DataSetReader = importVariable;", + "\n Options = options;", "\n Id = Guid.NewGuid();", "\n }", "\n", - "\n public KeyedStreamImport(){}", + "\n public KeyedStreamImport(Guid id)", + "\n {", + "\n Id = id;", + "\n Options = null;", + "\n DataSetReader = null;", + "\n }", "\n ", "\n}" ], @@ -261,7 +288,8 @@ "source": [ "public static async Task ExecuteWithStoreActivityAsync(this ImportOptionsBuilder builder, ", "\n ISessionVariable session, ", - "\n IDataSource dataSource)", + "\n IDataSource dataSource, ", + "\n IDataSetImportVariable importVariable)", "\n{", "\n var log = await builder.ExecuteAsync();", "\n var options = builder.GetImportOptions();", @@ -269,25 +297,25 @@ "\n switch(options)", "\n {", "\n case FileImportOptions fio:", - "\n var keyedFile = await (new KeyedFile(fio)).InitializeImportDataAsync(session) as KeyedFile;", + "\n var keyedFile = await (new KeyedFile(fio, importVariable)).InitializeImportDataAsync(session) as KeyedFile;", "\n activity = activity with {SourceId = keyedFile.Id, ", "\n Category = \"Import from File\"};", "\n await dataSource.UpdateAsync(keyedFile.RepeatOnce());", "\n break;", "\n case StringImportOptions sgio:", - "\n var keyedStringImport = await (new KeyedStringImport(sgio)).InitializeImportDataAsync(session) as KeyedStringImport;", + "\n var keyedStringImport = await (new KeyedStringImport(sgio, importVariable)).InitializeImportDataAsync(session) as KeyedStringImport;", "\n activity = activity with {SourceId = keyedStringImport.Id, ", "\n Category = \"Import from String\"};", "\n await dataSource.UpdateAsync(keyedStringImport.RepeatOnce());", "\n break;", "\n case StreamImportOptions smio:", - "\n var keyedStreamImport = await (new KeyedStreamImport(smio)).InitializeImportDataAsync(session) as KeyedStreamImport;", + "\n var keyedStreamImport = await (new KeyedStreamImport(smio, importVariable)).InitializeImportDataAsync(session) as KeyedStreamImport;", "\n activity = activity with {SourceId = keyedStreamImport.Id, ", "\n Category = \"Import from Stream\"};", "\n await dataSource.UpdateAsync(keyedStreamImport.RepeatOnce());", "\n break;", "\n case DataSetImportOptions dsio:", - "\n var keyedDataSetImport = await (new KeyedDataSetImport(dsio)).InitializeImportDataAsync(session) as KeyedDataSetImport;", + "\n var keyedDataSetImport = await (new KeyedDataSetImport(dsio, importVariable)).InitializeImportDataAsync(session) as KeyedDataSetImport;", "\n activity = activity with {SourceId = keyedDataSetImport.Id, ", "\n Category = \"Import from Data Set\"};", "\n await dataSource.UpdateAsync(keyedDataSetImport.RepeatOnce());", From 7414ae177a09ed5a9c4425c80bce7176704ba40d Mon Sep 17 00:00:00 2001 From: akatz Date: Fri, 6 Jan 2023 14:09:34 +0100 Subject: [PATCH 13/37] dealing with the exceptions if the memory stream cannot be produced (invalid input or missing file / stream / string). Most likely the final version modulo retrieving the partitions --- ifrs17/Utils/ActivityLogsAndImports.ipynb | 49 +++++++++++++++++++---- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/ifrs17/Utils/ActivityLogsAndImports.ipynb b/ifrs17/Utils/ActivityLogsAndImports.ipynb index 12e96868..172408aa 100644 --- a/ifrs17/Utils/ActivityLogsAndImports.ipynb +++ b/ifrs17/Utils/ActivityLogsAndImports.ipynb @@ -294,31 +294,64 @@ "\n var log = await builder.ExecuteAsync();", "\n var options = builder.GetImportOptions();", "\n var activity = new KeyedActivity(log, session);", + "\n bool importSucceeded = true;", "\n switch(options)", "\n {", "\n case FileImportOptions fio:", - "\n var keyedFile = await (new KeyedFile(fio, importVariable)).InitializeImportDataAsync(session) as KeyedFile;", + "\n var keyedFile = new KeyedFile(Guid.NewGuid());", + "\n try", + "\n {", + "\n keyedFile = await (new KeyedFile(fio, importVariable)).InitializeImportDataAsync(session) as KeyedFile;", + "\n }", + "\n catch (Exception)", + "\n {", + "\n importSucceeded = false;", + "\n }", "\n activity = activity with {SourceId = keyedFile.Id, ", "\n Category = \"Import from File\"};", - "\n await dataSource.UpdateAsync(keyedFile.RepeatOnce());", + "\n if (importSucceeded) await dataSource.UpdateAsync(keyedFile.RepeatOnce());", "\n break;", "\n case StringImportOptions sgio:", - "\n var keyedStringImport = await (new KeyedStringImport(sgio, importVariable)).InitializeImportDataAsync(session) as KeyedStringImport;", + "\n var keyedStringImport = new KeyedStringImport(Guid.NewGuid());", + "\n try", + "\n { ", + "\n keyedStringImport = await (new KeyedStringImport(sgio, importVariable)).InitializeImportDataAsync(session) as KeyedStringImport;", + "\n }", + "\n catch (Exception)", + "\n {", + "\n importSucceeded = false;", + "\n }", "\n activity = activity with {SourceId = keyedStringImport.Id, ", "\n Category = \"Import from String\"};", - "\n await dataSource.UpdateAsync(keyedStringImport.RepeatOnce());", + "\n if (importSucceeded) await dataSource.UpdateAsync(keyedStringImport.RepeatOnce());", "\n break;", "\n case StreamImportOptions smio:", - "\n var keyedStreamImport = await (new KeyedStreamImport(smio, importVariable)).InitializeImportDataAsync(session) as KeyedStreamImport;", + "\n var keyedStreamImport = new KeyedStreamImport(Guid.NewGuid());", + "\n try", + "\n {", + "\n keyedStreamImport = await (new KeyedStreamImport(smio, importVariable)).InitializeImportDataAsync(session) as KeyedStreamImport;", + "\n }", + "\n catch (Exception)", + "\n {", + "\n importSucceeded = false;", + "\n }", "\n activity = activity with {SourceId = keyedStreamImport.Id, ", "\n Category = \"Import from Stream\"};", - "\n await dataSource.UpdateAsync(keyedStreamImport.RepeatOnce());", + "\n if (importSucceeded) await dataSource.UpdateAsync(keyedStreamImport.RepeatOnce());", "\n break;", "\n case DataSetImportOptions dsio:", - "\n var keyedDataSetImport = await (new KeyedDataSetImport(dsio, importVariable)).InitializeImportDataAsync(session) as KeyedDataSetImport;", + "\n var keyedDataSetImport = new KeyedDataSetImport(Guid.NewGuid());", + "\n try", + "\n {", + "\n keyedDataSetImport = await (new KeyedDataSetImport(dsio, importVariable)).InitializeImportDataAsync(session) as KeyedDataSetImport;", + "\n }", + "\n catch(Exception)", + "\n {", + "\n importSucceeded = false;", + "\n }", "\n activity = activity with {SourceId = keyedDataSetImport.Id, ", "\n Category = \"Import from Data Set\"};", - "\n await dataSource.UpdateAsync(keyedDataSetImport.RepeatOnce());", + "\n if (importSucceeded) await dataSource.UpdateAsync(keyedDataSetImport.RepeatOnce());", "\n break;", "\n default:", "\n throw new Exception(\"Import Options object is not an instance of an appropriate class.\");", From 7e35f34050a178bfb18cba557c83ab4d55ea489f Mon Sep 17 00:00:00 2001 From: akatz Date: Fri, 6 Jan 2023 14:25:49 +0100 Subject: [PATCH 14/37] versions update --- ifrs17/DataModel/DataStructure.ipynb | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ifrs17/DataModel/DataStructure.ipynb b/ifrs17/DataModel/DataStructure.ipynb index c7cf2366..3b5ea261 100644 --- a/ifrs17/DataModel/DataStructure.ipynb +++ b/ifrs17/DataModel/DataStructure.ipynb @@ -52,21 +52,21 @@ { "cell_type": "code", "source": [ - "#r \"nuget:Systemorph.Activities,1.6.1\"", - "\n#r \"nuget:Systemorph.Arithmetics,1.6.1\"", + "#r \"nuget:Systemorph.Activities,1.6.2\"", + "\n#r \"nuget:Systemorph.Arithmetics,1.6.2\"", "\n#r \"nuget:Systemorph.Workspace,1.6.0\"", - "\n#r \"nuget:Systemorph.InteractiveObjects,1.6.1\"", - "\n#r \"nuget:Systemorph.SharePoint,1.6.1\"", - "\n#r \"nuget:Systemorph.OneDrive,1.6.1\"", - "\n#r \"nuget:Systemorph.Scopes,1.6.1\"", - "\n#r \"nuget:Systemorph.Import,1.6.3-ci-20230103-150150\"", - "\n#r \"nuget:Systemorph.Test,1.6.1\"", - "\n#r \"nuget:Systemorph.Export,1.6.3-ci-20230103-150150\"", - "\n#r \"nuget:Systemorph.DataSetReader,1.6.3-ci-20230103-150150\"", + "\n#r \"nuget:Systemorph.InteractiveObjects,1.6.2\"", + "\n#r \"nuget:Systemorph.SharePoint,1.6.2\"", + "\n#r \"nuget:Systemorph.OneDrive,1.6.2\"", + "\n#r \"nuget:Systemorph.Scopes,1.6.2\"", + "\n#r \"nuget:Systemorph.Import,1.6.3\"", + "\n#r \"nuget:Systemorph.Test,1.6.2\"", + "\n#r \"nuget:Systemorph.Export,1.6.3\"", + "\n#r \"nuget:Systemorph.DataSetReader,1.6.3\"", "\n#r \"nuget:Systemorph.DataSource,1.6.0\"", "\n#r \"nuget:Systemorph.DataSource.Conversions,1.6.0\"", - "\n#r \"nuget:Systemorph.Reporting,1.6.1\"", - "\n#r \"nuget:Systemorph.Charting,1.6.1\"", + "\n#r \"nuget:Systemorph.Reporting,1.6.2\"", + "\n#r \"nuget:Systemorph.Charting,1.6.2\"", "\n#r \"nuget:Systemorph.SchemaMigrations,1.6.1\"" ], "metadata": {}, From 11e8bb9a919e220f8164994753dbddb8f4d56275 Mon Sep 17 00:00:00 2001 From: akatz Date: Fri, 6 Jan 2023 17:56:30 +0100 Subject: [PATCH 15/37] update CalculationEngine import --- ifrs17/CalculationEngine.ipynb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ifrs17/CalculationEngine.ipynb b/ifrs17/CalculationEngine.ipynb index 288ee9fb..1cb684b2 100644 --- a/ifrs17/CalculationEngine.ipynb +++ b/ifrs17/CalculationEngine.ipynb @@ -22,7 +22,8 @@ "\n#!import \"Report/ReportMutableScopes\"", "\n#!import \"Import/Importers\"", "\n#!import \"Export/ExportConfiguration\"", - "\n#!import \"Utils/TestHelper\"" + "\n#!import \"Utils/TestHelper\"", + "\n#!import \"Utils/ActivityLogsAndImports\"" ], "metadata": {}, "execution_count": 0, From 422368ad1fd04f6804918aae82dc2986fa575383 Mon Sep 17 00:00:00 2001 From: akatz Date: Fri, 6 Jan 2023 18:15:58 +0100 Subject: [PATCH 16/37] update imports in initialization of ref data to memory --- .../InitSystemorphRefDataToMemory.ipynb | 67 +++++++++++++++---- 1 file changed, 54 insertions(+), 13 deletions(-) diff --git a/ifrs17-template/Initialization/InitSystemorphRefDataToMemory.ipynb b/ifrs17-template/Initialization/InitSystemorphRefDataToMemory.ipynb index f4c5b6f8..b0fb3cba 100644 --- a/ifrs17-template/Initialization/InitSystemorphRefDataToMemory.ipynb +++ b/ifrs17-template/Initialization/InitSystemorphRefDataToMemory.ipynb @@ -118,7 +118,7 @@ "\n .WithType()", "\n .WithType()", "\n .WithTarget(DataSource)", - "\n .ExecuteAsync()" + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -127,7 +127,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/Dimensions.csv\").WithFormat(ImportFormats.AocConfiguration).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/Dimensions.csv\")", + "\n .WithFormat(ImportFormats.AocConfiguration)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -136,7 +139,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/ReportingNodes/ReportingNodes.csv\").WithType().WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/ReportingNodes/ReportingNodes.csv\")", + "\n .WithType()", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -154,13 +160,39 @@ { "cell_type": "code", "source": [ - "var log = await Import.FromFile(\"../Files/Parameters/YieldCurve_2019_12.csv\").WithFormat(ImportFormats.YieldCurve).WithTarget(DataSource).ExecuteAsync();", - "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2020_1.csv\").WithFormat(ImportFormats.YieldCurve).WithTarget(DataSource).ExecuteAsync());", - "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2020_3.csv\").WithFormat(ImportFormats.YieldCurve).WithTarget(DataSource).ExecuteAsync());", - "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2020_12.csv\").WithFormat(ImportFormats.YieldCurve).WithTarget(DataSource).ExecuteAsync());", - "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2021_3.csv\").WithFormat(ImportFormats.YieldCurve).WithTarget(DataSource).ExecuteAsync());", - "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2021_6.csv\").WithFormat(ImportFormats.YieldCurve).WithTarget(DataSource).ExecuteAsync());", - "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2021_12.csv\").WithFormat(ImportFormats.YieldCurve).WithTarget(DataSource).ExecuteAsync());", + "var log = await Import.FromFile(\"../Files/Parameters/YieldCurve_2019_12.csv\")", + "\n .WithFormat(ImportFormats.YieldCurve)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader);", + "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2020_1.csv\")", + "\n .WithFormat(ImportFormats.YieldCurve)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)", + "\n );", + "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2020_3.csv\")", + "\n .WithFormat(ImportFormats.YieldCurve)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)", + "\n );", + "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2020_12.csv\")", + "\n .WithFormat(ImportFormats.YieldCurve)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)", + "\n );", + "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2021_3.csv\")", + "\n .WithFormat(ImportFormats.YieldCurve)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)", + "\n );", + "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2021_6.csv\")", + "\n .WithFormat(ImportFormats.YieldCurve)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)", + "\n );", + "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2021_12.csv\")", + "\n .WithFormat(ImportFormats.YieldCurve)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader));", "\nlog" ], "metadata": {}, @@ -170,7 +202,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/Parameters/ExchangeRate.csv\").WithType().WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/Parameters/ExchangeRate.csv\")", + "\n .WithType()", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -179,7 +214,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/Parameters/PartnerRating.csv\").WithType().WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/Parameters/PartnerRating.csv\")", + "\n .WithType()", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -188,7 +226,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/Parameters/CreditDefaultRate.csv\").WithType().WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/Parameters/CreditDefaultRate.csv\")", + "\n .WithType()", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, From cae0f57b05ab23defa6295d75d8ead96f88113ed Mon Sep 17 00:00:00 2001 From: akatz Date: Mon, 9 Jan 2023 09:19:01 +0100 Subject: [PATCH 17/37] imports fixed --- .../InitSystemorphBaseToMemory.ipynb | 15 ++- .../InitSystemorphToDatabase.ipynb | 111 ++++++++++++++---- .../InitSystemorphToMemory.ipynb | 35 ++++-- 3 files changed, 130 insertions(+), 31 deletions(-) diff --git a/ifrs17-template/Initialization/InitSystemorphBaseToMemory.ipynb b/ifrs17-template/Initialization/InitSystemorphBaseToMemory.ipynb index 92447054..84f27378 100644 --- a/ifrs17-template/Initialization/InitSystemorphBaseToMemory.ipynb +++ b/ifrs17-template/Initialization/InitSystemorphBaseToMemory.ipynb @@ -77,7 +77,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/DataNodes/DataNodes_CH.csv\").WithFormat(ImportFormats.DataNode).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/DataNodes/DataNodes_CH.csv\")", + "\n .WithFormat(ImportFormats.DataNode)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -86,7 +89,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/DataNodes/DataNodeStates_CH_2020_12.csv\").WithFormat(ImportFormats.DataNodeState).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/DataNodes/DataNodeStates_CH_2020_12.csv\")", + "\n .WithFormat(ImportFormats.DataNodeState)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -95,7 +101,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/DataNodes/DataNodeParameters_CH_2020_12.csv\").WithFormat(ImportFormats.DataNodeParameter).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/DataNodes/DataNodeParameters_CH_2020_12.csv\")", + "\n .WithFormat(ImportFormats.DataNodeParameter)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, diff --git a/ifrs17-template/Initialization/InitSystemorphToDatabase.ipynb b/ifrs17-template/Initialization/InitSystemorphToDatabase.ipynb index a8aea982..79af62d4 100644 --- a/ifrs17-template/Initialization/InitSystemorphToDatabase.ipynb +++ b/ifrs17-template/Initialization/InitSystemorphToDatabase.ipynb @@ -105,7 +105,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/ReportingNodes/ReportingNodes.csv\").WithType().WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/ReportingNodes/ReportingNodes.csv\")", + "\n .WithType()", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -123,13 +126,40 @@ { "cell_type": "code", "source": [ - "var log = await Import.FromFile(\"../Files/Parameters/YieldCurve_2019_12.csv\").WithFormat(ImportFormats.YieldCurve).WithTarget(DataSource).ExecuteAsync();", - "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2020_1.csv\").WithFormat(ImportFormats.YieldCurve).WithTarget(DataSource).ExecuteAsync());", - "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2020_3.csv\").WithFormat(ImportFormats.YieldCurve).WithTarget(DataSource).ExecuteAsync());", - "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2020_12.csv\").WithFormat(ImportFormats.YieldCurve).WithTarget(DataSource).ExecuteAsync());", - "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2021_3.csv\").WithFormat(ImportFormats.YieldCurve).WithTarget(DataSource).ExecuteAsync());", - "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2021_6.csv\").WithFormat(ImportFormats.YieldCurve).WithTarget(DataSource).ExecuteAsync());", - "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2021_12.csv\").WithFormat(ImportFormats.YieldCurve).WithTarget(DataSource).ExecuteAsync());", + "var log = await Import.FromFile(\"../Files/Parameters/YieldCurve_2019_12.csv\")", + "\n .WithFormat(ImportFormats.YieldCurve)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader);", + "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2020_1.csv\")", + "\n .WithFormat(ImportFormats.YieldCurve)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)", + "\n );", + "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2020_3.csv\")", + "\n .WithFormat(ImportFormats.YieldCurve)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)", + "\n );", + "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2020_12.csv\")", + "\n .WithFormat(ImportFormats.YieldCurve)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)", + "\n );", + "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2021_3.csv\")", + "\n .WithFormat(ImportFormats.YieldCurve)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)", + "\n );", + "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2021_6.csv\")", + "\n .WithFormat(ImportFormats.YieldCurve)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)", + "\n );", + "\nlog.Merge(await Import.FromFile(\"../Files/Parameters/YieldCurve_2021_12.csv\")", + "\n .WithFormat(ImportFormats.YieldCurve)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)", + "\n );", "\nlog" ], "metadata": {}, @@ -139,7 +169,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/Parameters/ExchangeRate.csv\").WithType().WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/Parameters/ExchangeRate.csv\")", + "\n .WithType()", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -148,7 +181,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/Parameters/PartnerRating.csv\").WithType().WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/Parameters/PartnerRating.csv\")", + "\n .WithType()", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -157,7 +193,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/Parameters/CreditDefaultRate.csv\").WithType().WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/Parameters/CreditDefaultRate.csv\")", + "\n .WithType()", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -175,7 +214,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/DataNodes/DataNodes_CH.csv\").WithFormat(ImportFormats.DataNode).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/DataNodes/DataNodes_CH.csv\")", + "\n .WithFormat(ImportFormats.DataNode)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -184,7 +226,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/DataNodes/DataNodeStates_CH_2020_12.csv\").WithFormat(ImportFormats.DataNodeState).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/DataNodes/DataNodeStates_CH_2020_12.csv\")", + "\n .WithFormat(ImportFormats.DataNodeState)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -193,7 +238,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/DataNodes/DataNodeParameters_CH_2020_12.csv\").WithFormat(ImportFormats.DataNodeParameter).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/DataNodes/DataNodeParameters_CH_2020_12.csv\")", + "\n .WithFormat(ImportFormats.DataNodeParameter)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -211,7 +259,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/TransactionalData/Openings_CH_2020_12.csv\").WithFormat(ImportFormats.Opening).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/TransactionalData/Openings_CH_2020_12.csv\")", + "\n .WithFormat(ImportFormats.Opening)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -220,7 +271,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/TransactionalData/NominalCashflows_CH_2020_12.csv\").WithFormat(ImportFormats.Cashflow).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/TransactionalData/NominalCashflows_CH_2020_12.csv\")", + "\n .WithFormat(ImportFormats.Cashflow)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -229,7 +283,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/TransactionalData/Actuals_CH_2020_12.csv\").WithFormat(ImportFormats.Actual).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/TransactionalData/Actuals_CH_2020_12.csv\")", + "\n .WithFormat(ImportFormats.Actual)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -238,7 +295,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/TransactionalData/NominalCashflows_CH_2021_3.csv\").WithFormat(ImportFormats.Cashflow).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/TransactionalData/NominalCashflows_CH_2021_3.csv\")", + "\n .WithFormat(ImportFormats.Cashflow)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -247,7 +307,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/TransactionalData/Actuals_CH_2021_3.csv\").WithFormat(ImportFormats.Actual).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/TransactionalData/Actuals_CH_2021_3.csv\")", + "\n .WithFormat(ImportFormats.Actual)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -256,7 +319,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/TransactionalData/SimpleValue_CH_2020_12.csv\").WithFormat(ImportFormats.SimpleValue ).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/TransactionalData/SimpleValue_CH_2020_12.csv\")", + "\n .WithFormat(ImportFormats.SimpleValue )", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -265,7 +331,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/TransactionalData/NominalCashflows_CH_2020_12_MTUP10pct.csv\").WithFormat(ImportFormats.Cashflow).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/TransactionalData/NominalCashflows_CH_2020_12_MTUP10pct.csv\")", + "\n .WithFormat(ImportFormats.Cashflow)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, diff --git a/ifrs17-template/Initialization/InitSystemorphToMemory.ipynb b/ifrs17-template/Initialization/InitSystemorphToMemory.ipynb index f349f687..d82e0186 100644 --- a/ifrs17-template/Initialization/InitSystemorphToMemory.ipynb +++ b/ifrs17-template/Initialization/InitSystemorphToMemory.ipynb @@ -67,7 +67,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/TransactionalData/Openings_CH_2020_12.csv\").WithFormat(ImportFormats.Opening).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/TransactionalData/Openings_CH_2020_12.csv\")", + "\n .WithFormat(ImportFormats.Opening)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -76,7 +79,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/TransactionalData/NominalCashflows_CH_2020_12.csv\").WithFormat(ImportFormats.Cashflow).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/TransactionalData/NominalCashflows_CH_2020_12.csv\")", + "\n .WithFormat(ImportFormats.Cashflow)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -85,7 +91,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/TransactionalData/Actuals_CH_2020_12.csv\").WithFormat(ImportFormats.Actual).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/TransactionalData/Actuals_CH_2020_12.csv\")", + "\n .WithFormat(ImportFormats.Actual)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -94,7 +103,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/TransactionalData/NominalCashflows_CH_2021_3.csv\").WithFormat(ImportFormats.Cashflow).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/TransactionalData/NominalCashflows_CH_2021_3.csv\")", + "\n .WithFormat(ImportFormats.Cashflow)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -103,7 +115,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/TransactionalData/Actuals_CH_2021_3.csv\").WithFormat(ImportFormats.Actual).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/TransactionalData/Actuals_CH_2021_3.csv\")", + "\n .WithFormat(ImportFormats.Actual)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -112,7 +127,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/TransactionalData/SimpleValue_CH_2020_12.csv\").WithFormat(ImportFormats.SimpleValue ).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/TransactionalData/SimpleValue_CH_2020_12.csv\")", + "\n .WithFormat(ImportFormats.SimpleValue )", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -121,7 +139,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/TransactionalData/NominalCashflows_CH_2020_12_MTUP10pct.csv\").WithFormat(ImportFormats.Cashflow).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"../Files/TransactionalData/NominalCashflows_CH_2020_12_MTUP10pct.csv\")", + "\n .WithFormat(ImportFormats.Cashflow)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, From 457abd2f8a719666ee9cc7ba98e246fe557fd4e1 Mon Sep 17 00:00:00 2001 From: akatz Date: Mon, 9 Jan 2023 09:22:32 +0100 Subject: [PATCH 18/37] import templates --- ifrs17-template/Import/CloseImportTemplate.ipynb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ifrs17-template/Import/CloseImportTemplate.ipynb b/ifrs17-template/Import/CloseImportTemplate.ipynb index 1f9b84ac..aa95ca1d 100644 --- a/ifrs17-template/Import/CloseImportTemplate.ipynb +++ b/ifrs17-template/Import/CloseImportTemplate.ipynb @@ -118,7 +118,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(pathToFile).WithFormat(format).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(pathToFile)", + "\n .WithFormat(format)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -163,7 +166,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(pathToFile).WithFormat(format).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(pathToFile)", + "\n .WithFormat(format)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, From bad7e198c11cd38d50996d1f1a0ff2a4081e97ae Mon Sep 17 00:00:00 2001 From: akatz Date: Mon, 9 Jan 2023 09:51:47 +0100 Subject: [PATCH 19/37] use cases update --- .../ActualsUseCaseDataImport.ipynb | 45 +++++++++++++++---- .../CsmSwitchDataImport.ipynb | 45 +++++++++++++++---- 2 files changed, 72 insertions(+), 18 deletions(-) diff --git a/ifrs17-template/PracticalUseCases/ActualsOutsideThePeriod/ActualsUseCaseDataImport.ipynb b/ifrs17-template/PracticalUseCases/ActualsOutsideThePeriod/ActualsUseCaseDataImport.ipynb index 04c02bc4..5db058d3 100644 --- a/ifrs17-template/PracticalUseCases/ActualsOutsideThePeriod/ActualsUseCaseDataImport.ipynb +++ b/ifrs17-template/PracticalUseCases/ActualsOutsideThePeriod/ActualsUseCaseDataImport.ipynb @@ -74,7 +74,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"DataNodes_ActualsCase_CH.csv\").WithFormat(ImportFormats.DataNode).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"DataNodes_ActualsCase_CH.csv\")", + "\n .WithFormat(ImportFormats.DataNode)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -83,7 +86,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"DataNodeStates_ActualsCase_CH_2020_1.csv\").WithFormat(ImportFormats.DataNodeState).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"DataNodeStates_ActualsCase_CH_2020_1.csv\")", + "\n .WithFormat(ImportFormats.DataNodeState)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -92,7 +98,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"DataNodeParameters_ActualsCase_CH_2020_1.csv\").WithFormat(ImportFormats.DataNodeParameter).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"DataNodeParameters_ActualsCase_CH_2020_1.csv\")", + "\n .WithFormat(ImportFormats.DataNodeParameter)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -119,7 +128,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"NominalCashflows_ActualsCase_CH_2020_12.csv\").WithFormat(ImportFormats.Cashflow).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"NominalCashflows_ActualsCase_CH_2020_12.csv\")", + "\n .WithFormat(ImportFormats.Cashflow)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -128,7 +140,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"Actuals_ActualsCase_CH_2020_12.csv\").WithFormat(ImportFormats.Actual).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"Actuals_ActualsCase_CH_2020_12.csv\")", + "\n .WithFormat(ImportFormats.Actual)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -146,7 +161,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"NominalCashflows_ActualsCase_CH_2021_6.csv\").WithFormat(ImportFormats.Cashflow).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"NominalCashflows_ActualsCase_CH_2021_6.csv\")", + "\n .WithFormat(ImportFormats.Cashflow)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -155,7 +173,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"Actuals_ActualsCase_CH_2021_6.csv\").WithFormat(ImportFormats.Actual).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"Actuals_ActualsCase_CH_2021_6.csv\")", + "\n .WithFormat(ImportFormats.Actual)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -173,7 +194,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"NominalCashflows_ActualsCase_CH_2021_12.csv\").WithFormat(ImportFormats.Cashflow).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"NominalCashflows_ActualsCase_CH_2021_12.csv\")", + "\n .WithFormat(ImportFormats.Cashflow)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -182,7 +206,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"Actuals_ActualsCase_CH_2021_12.csv\").WithFormat(ImportFormats.Actual).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"Actuals_ActualsCase_CH_2021_12.csv\")", + "\n .WithFormat(ImportFormats.Actual)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, diff --git a/ifrs17-template/PracticalUseCases/SingleVsMultipleCsmSwitch/CsmSwitchDataImport.ipynb b/ifrs17-template/PracticalUseCases/SingleVsMultipleCsmSwitch/CsmSwitchDataImport.ipynb index 8e7be7b6..a8493d9b 100644 --- a/ifrs17-template/PracticalUseCases/SingleVsMultipleCsmSwitch/CsmSwitchDataImport.ipynb +++ b/ifrs17-template/PracticalUseCases/SingleVsMultipleCsmSwitch/CsmSwitchDataImport.ipynb @@ -73,7 +73,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"DataNodes_CsmSwitch_CH.csv\").WithFormat(ImportFormats.DataNode).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"DataNodes_CsmSwitch_CH.csv\")", + "\n .WithFormat(ImportFormats.DataNode)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -82,7 +85,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"DataNodeStates_CsmSwitch_CH_2020_12.csv\").WithFormat(ImportFormats.DataNodeState).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"DataNodeStates_CsmSwitch_CH_2020_12.csv\")", + "\n .WithFormat(ImportFormats.DataNodeState)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -91,7 +97,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"DataNodeParameters_CsmSwitch_CH_2020_12.csv\").WithFormat(ImportFormats.DataNodeParameter).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"DataNodeParameters_CsmSwitch_CH_2020_12.csv\")", + "\n .WithFormat(ImportFormats.DataNodeParameter)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -118,7 +127,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"NominalCashflows_CsmSwitch_CH_2020_12.csv\").WithFormat(ImportFormats.Cashflow).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"NominalCashflows_CsmSwitch_CH_2020_12.csv\")", + "\n .WithFormat(ImportFormats.Cashflow)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -127,7 +139,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"Actuals_CsmSwitch_CH_2020_12.csv\").WithFormat(ImportFormats.Actual).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"Actuals_CsmSwitch_CH_2020_12.csv\")", + "\n .WithFormat(ImportFormats.Actual)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -136,7 +151,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"NominalCashflows_CsmSwitch_CH_2021_3.csv\").WithFormat(ImportFormats.Cashflow).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"NominalCashflows_CsmSwitch_CH_2021_3.csv\")", + "\n .WithFormat(ImportFormats.Cashflow)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -145,7 +163,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"Actuals_CsmSwitch_CH_2021_3.csv\").WithFormat(ImportFormats.Actual).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"Actuals_CsmSwitch_CH_2021_3.csv\")", + "\n .WithFormat(ImportFormats.Actual)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -163,7 +184,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"SimpleValue_CsmSwitch_CH_2020_12.csv\").WithFormat(ImportFormats.SimpleValue).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"SimpleValue_CsmSwitch_CH_2020_12.csv\")", + "\n .WithFormat(ImportFormats.SimpleValue)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -172,7 +196,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"SimpleValue_CsmSwitch_CH_2021_3.csv\").WithFormat(ImportFormats.SimpleValue).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"SimpleValue_CsmSwitch_CH_2021_3.csv\")", + "\n .WithFormat(ImportFormats.SimpleValue)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, From 03412406312fbfedce728de89d42174c77f75394 Mon Sep 17 00:00:00 2001 From: akatz Date: Tue, 10 Jan 2023 15:14:38 +0100 Subject: [PATCH 20/37] version change --- ifrs17/DataModel/DataStructure.ipynb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ifrs17/DataModel/DataStructure.ipynb b/ifrs17/DataModel/DataStructure.ipynb index 69076f5f..12db6b17 100644 --- a/ifrs17/DataModel/DataStructure.ipynb +++ b/ifrs17/DataModel/DataStructure.ipynb @@ -54,15 +54,15 @@ "source": [ "#r \"nuget:Systemorph.Activities,1.6.2\"", "\n#r \"nuget:Systemorph.Arithmetics,1.6.2\"", - "\n#r \"nuget:Systemorph.Workspace,1.6.0\"", + "\n#r \"nuget:Systemorph.Workspace,1.6.1\"", "\n#r \"nuget:Systemorph.InteractiveObjects,1.6.2\"", "\n#r \"nuget:Systemorph.SharePoint,1.6.2\"", "\n#r \"nuget:Systemorph.OneDrive,1.6.2\"", "\n#r \"nuget:Systemorph.Scopes,1.6.2\"", - "\n#r \"nuget:Systemorph.Import,1.6.3\"", + "\n#r \"nuget:Systemorph.Import,1.6.4-ci-20230110-092155\"", "\n#r \"nuget:Systemorph.Test,1.6.2\"", - "\n#r \"nuget:Systemorph.Export,1.6.3\"", - "\n#r \"nuget:Systemorph.DataSetReader,1.6.3\"", + "\n#r \"nuget:Systemorph.Export,1.6.4-ci-20230110-092155\"", + "\n#r \"nuget:Systemorph.DataSetReader,1.6.4-ci-20230110-092155\"", "\n#r \"nuget:Systemorph.DataSource,1.6.0\"", "\n#r \"nuget:Systemorph.DataSource.Conversions,1.6.0\"", "\n#r \"nuget:Systemorph.Reporting,1.6.2\"", From ee236a589a7fdbb75f428b42d964c154e07b2156 Mon Sep 17 00:00:00 2001 From: akatz Date: Tue, 10 Jan 2023 15:20:47 +0100 Subject: [PATCH 21/37] rename + imports --- ifrs17/Utils/ActivityLogsAndImports.ipynb | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/ifrs17/Utils/ActivityLogsAndImports.ipynb b/ifrs17/Utils/ActivityLogsAndImports.ipynb index 172408aa..4b1e1e3a 100644 --- a/ifrs17/Utils/ActivityLogsAndImports.ipynb +++ b/ifrs17/Utils/ActivityLogsAndImports.ipynb @@ -34,6 +34,9 @@ "\nusing Systemorph.Vertex.Import.Builders;", "\nusing Systemorph.Vertex.Import.Mappings;", "\nusing Systemorph.Vertex.DataSetReader;", + "\nusing Systemorph.Vertex.Export.Excel.Builders;", + "\nusing Systemorph.Vertex.Export;", + "\nusing Systemorph.Vertex.FileStorage;", "\n", "\npublic record StreamWrapper(Stream Stream, bool WillBeReused);", "\n", @@ -46,7 +49,7 @@ { "cell_type": "code", "source": [ - "public record KeyedActivity : KeyedRecord", + "public record ImportExportActivityActivity : KeyedRecord", "\n{", "\n public string Username {get; init;}", "\n", @@ -69,7 +72,7 @@ "\n", "\n public Guid SourceId {get; init;} // Check if converting to Guid causes issues. If not, leave Guid.", "\n", - "\n public KeyedActivity(ActivityLog log, ISessionVariable session)", + "\n public ImportExportActivity(ActivityLog log, ISessionVariable session)", "\n {", "\n Id = Guid.NewGuid();", "\n Username = session.User.Name;", @@ -81,7 +84,7 @@ "\n InfoMessages = log.Infos.Select(x => x.ProcessNotification()).Distinct().ToArray();", "\n }", "\n", - "\n public KeyedActivity(Guid id)", + "\n public ImportExportActivity(Guid id)", "\n {", "\n Id = id;", "\n }", @@ -92,6 +95,15 @@ "execution_count": 0, "outputs": [] }, + { + "cell_type": "code", + "source": [ + "" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, { "cell_type": "code", "source": [ From d7568f9f723d787407787c4a78e1806707fe6647 Mon Sep 17 00:00:00 2001 From: akatz Date: Tue, 10 Jan 2023 15:35:37 +0100 Subject: [PATCH 22/37] new abstraction level and renamings --- ifrs17/Utils/ActivityLogsAndImports.ipynb | 38 +++++++++++++---------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/ifrs17/Utils/ActivityLogsAndImports.ipynb b/ifrs17/Utils/ActivityLogsAndImports.ipynb index 4b1e1e3a..46409f62 100644 --- a/ifrs17/Utils/ActivityLogsAndImports.ipynb +++ b/ifrs17/Utils/ActivityLogsAndImports.ipynb @@ -49,7 +49,7 @@ { "cell_type": "code", "source": [ - "public record ImportExportActivityActivity : KeyedRecord", + "public record ImportExportActivity : KeyedRecord", "\n{", "\n public string Username {get; init;}", "\n", @@ -98,7 +98,15 @@ { "cell_type": "code", "source": [ - "" + "public abstract record KeyedImportExport : KeyedRecord", + "\n{", + "\n public DateTime CreationTime {get; init;}", + "\n public byte[] SerializedContent {get; init;}", + "\n public uint? Length {get; init;}", + "\n public string Format {get; init;}", + "\n protected IDataSetImportVariable DataSetReader {get; set;}", + "\n protected ISessionVariable Session {get; set;}", + "\n}" ], "metadata": {}, "execution_count": 0, @@ -107,19 +115,14 @@ { "cell_type": "code", "source": [ - "public abstract record ImportWithKey : KeyedRecord", + "public abstract record KeyedImport : KeyedImportExport", "\n{", - "\n public DateTime CreationTime {get; init;}", - "\n public byte[] SerializedContent {get; init;}", - "\n public uint? Length {get; init;}", - "\n public string Format {get; init;}", "\n protected ImportOptions Options {get; set;}", - "\n protected IDataSetImportVariable DataSetReader{get; set;}", "\n", - "\n public async Task InitializeImportDataAsync(ISessionVariable session)", + "\n public async Task InitializeImportDataAsync()", "\n {", - "\n var stream = await GenerateStreamWrapperAsync(session);", - "\n var formatAndContent = await GetInformationFromStreamAsync(stream, session);", + "\n var stream = await GenerateStreamWrapperAsync();", + "\n var formatAndContent = await GetInformationFromStreamAsync(stream);", "\n return this with{CreationTime = DateTime.UtcNow, ", "\n Format = formatAndContent.Format ?? Options.Format,", "\n SerializedContent = formatAndContent.Content,", @@ -127,11 +130,11 @@ "\n };", "\n }", "\n", - "\n private async Task GenerateStreamWrapperAsync(ISessionVariable session)", + "\n private async Task GenerateStreamWrapperAsync()", "\n {", "\n StreamWrapper stream = Options switch", "\n {", - "\n FileImportOptions fio => new StreamWrapper(await fio.Storage.ReadAsync(fio.FileName, session.CancellationToken), true),", + "\n FileImportOptions fio => new StreamWrapper(await fio.Storage.ReadAsync(fio.FileName, Session.CancellationToken), true),", "\n StreamImportOptions streamImportOptions => new StreamWrapper(streamImportOptions.Stream, false),", "\n StringImportOptions stringImportOptions => new StreamWrapper(new MemoryStream(Encoding.ASCII.GetBytes(stringImportOptions.Content)), true),", "\n DataSetImportOptions dataSetImportOptions => new StreamWrapper(new MemoryStream(Encoding.ASCII.GetBytes(DataSetCsvSerializer.Serialize(dataSetImportOptions.DataSet))), true),", @@ -140,7 +143,7 @@ "\n return stream;", "\n }", "\n", - "\n private async Task<(string Format, byte[] Content)> GetInformationFromStreamAsync(StreamWrapper stream, ISessionVariable session)", + "\n private async Task<(string Format, byte[] Content)> GetInformationFromStreamAsync(StreamWrapper stream)", "\n {", "\n byte[] content;", "\n string format;", @@ -170,7 +173,7 @@ { "cell_type": "code", "source": [ - "public record KeyedFile : ImportWithKey", + "public record ImportFile : KeyedImport", "\n{", "\n public string Name {get; init;}", "\n", @@ -183,10 +186,11 @@ "\n", "\n public string Source {get; init;}", "\n", - "\n public KeyedFile(FileImportOptions options, IDataSetImportVariable importVariable)", + "\n public ImportFile(FileImportOptions options, IDataSetImportVariable importVariable, ISessionVariable session)", "\n {", "\n Options = options;", "\n DataSetReader = importVariable;", + "\n Session = session;", "\n string fileName = options.FileName; ", "\n Id = Guid.NewGuid();", "\n Name = Path.GetFileName(fileName);", @@ -197,7 +201,7 @@ "\n // Andrey Katz: Options.TargetDataSource.Partion.GetCurrent(?? What do we put here, different classes might posess various partitions, e.g. Yield Curve has none ??)", "\n }", "\n", - "\n public KeyedFile(Guid id)", + "\n public ImportFile(Guid id)", "\n {", "\n Id = id;", "\n Options = null;", From 7b1da30b756c0f770100db7b4b5ec683fdc7a385 Mon Sep 17 00:00:00 2001 From: akatz Date: Tue, 10 Jan 2023 15:45:15 +0100 Subject: [PATCH 23/37] renamings --- ifrs17/Utils/ActivityLogsAndImports.ipynb | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/ifrs17/Utils/ActivityLogsAndImports.ipynb b/ifrs17/Utils/ActivityLogsAndImports.ipynb index 46409f62..bf43e5c4 100644 --- a/ifrs17/Utils/ActivityLogsAndImports.ipynb +++ b/ifrs17/Utils/ActivityLogsAndImports.ipynb @@ -224,19 +224,20 @@ { "cell_type": "code", "source": [ - "public record KeyedStringImport : ImportWithKey", + "public record ImportString : KeyedImport", "\n{", "\n public string Content {get; init;}", "\n", - "\n public KeyedStringImport(StringImportOptions options, IDataSetImportVariable importVariable)", + "\n public ImportString(StringImportOptions options, IDataSetImportVariable importVariable, ISessionVariable session)", "\n {", "\n Options = options;", "\n DataSetReader = importVariable;", + "\n Session = session;", "\n Id = Guid.NewGuid();", "\n Content = options.Content;", "\n }", "\n", - "\n public KeyedStringImport(Guid id)", + "\n public ImportString(Guid id)", "\n {", "\n Id = id;", "\n Options = null;", @@ -252,16 +253,17 @@ { "cell_type": "code", "source": [ - "public record KeyedDataSetImport : ImportWithKey", + "public record ImportDataSet : KeyedImport", "\n{", - "\n public KeyedDataSetImport(DataSetImportOptions options, IDataSetImportVariable importVariable)", + "\n public ImportDataSet(DataSetImportOptions options, IDataSetImportVariable importVariable, ISessionVariable session)", "\n {", + "\n Session = session;", "\n DataSetReader = importVariable;", "\n Options = options;", "\n Id = Guid.NewGuid();", "\n }", "\n", - "\n public KeyedDataSetImport(Guid id)", + "\n public ImportDataSet(Guid id)", "\n {", "\n Id = id;", "\n Options = null;", @@ -277,16 +279,17 @@ { "cell_type": "code", "source": [ - "public record KeyedStreamImport : ImportWithKey", + "public record ImportStream : KeyedImport", "\n{", - "\n public KeyedStreamImport(StreamImportOptions options, IDataSetImportVariable importVariable)", + "\n public ImportStream(StreamImportOptions options, IDataSetImportVariable importVariable, ISessionVariable session)", "\n {", + "\n Session = session;", "\n DataSetReader = importVariable;", "\n Options = options;", "\n Id = Guid.NewGuid();", "\n }", "\n", - "\n public KeyedStreamImport(Guid id)", + "\n public ImportStream(Guid id)", "\n {", "\n Id = id;", "\n Options = null;", From ec319062e2541271e807fbe044c04f8b7f1d62f3 Mon Sep 17 00:00:00 2001 From: akatz Date: Tue, 10 Jan 2023 16:04:29 +0100 Subject: [PATCH 24/37] renaming round done --- ifrs17/Utils/ActivityLogsAndImports.ipynb | 36 +++++++++++------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/ifrs17/Utils/ActivityLogsAndImports.ipynb b/ifrs17/Utils/ActivityLogsAndImports.ipynb index bf43e5c4..18171148 100644 --- a/ifrs17/Utils/ActivityLogsAndImports.ipynb +++ b/ifrs17/Utils/ActivityLogsAndImports.ipynb @@ -312,71 +312,71 @@ "\n{", "\n var log = await builder.ExecuteAsync();", "\n var options = builder.GetImportOptions();", - "\n var activity = new KeyedActivity(log, session);", + "\n var activity = new ImportExportActivity(log, session);", "\n bool importSucceeded = true;", "\n switch(options)", "\n {", "\n case FileImportOptions fio:", - "\n var keyedFile = new KeyedFile(Guid.NewGuid());", + "\n var importFile = new ImportFile(Guid.NewGuid());", "\n try", "\n {", - "\n keyedFile = await (new KeyedFile(fio, importVariable)).InitializeImportDataAsync(session) as KeyedFile;", + "\n importFile = await (new ImportFile(fio, importVariable, session)).InitializeImportDataAsync() as ImportFile;", "\n }", "\n catch (Exception)", "\n {", "\n importSucceeded = false;", "\n }", - "\n activity = activity with {SourceId = keyedFile.Id, ", + "\n activity = activity with {SourceId = importFile.Id, ", "\n Category = \"Import from File\"};", - "\n if (importSucceeded) await dataSource.UpdateAsync(keyedFile.RepeatOnce());", + "\n if (importSucceeded) await dataSource.UpdateAsync(importFile.RepeatOnce());", "\n break;", "\n case StringImportOptions sgio:", - "\n var keyedStringImport = new KeyedStringImport(Guid.NewGuid());", + "\n var importString = new ImportString(Guid.NewGuid());", "\n try", "\n { ", - "\n keyedStringImport = await (new KeyedStringImport(sgio, importVariable)).InitializeImportDataAsync(session) as KeyedStringImport;", + "\n importString = await (new ImportString(sgio, importVariable, session)).InitializeImportDataAsync() as ImportString;", "\n }", "\n catch (Exception)", "\n {", "\n importSucceeded = false;", "\n }", - "\n activity = activity with {SourceId = keyedStringImport.Id, ", + "\n activity = activity with {SourceId = importString.Id, ", "\n Category = \"Import from String\"};", - "\n if (importSucceeded) await dataSource.UpdateAsync(keyedStringImport.RepeatOnce());", + "\n if (importSucceeded) await dataSource.UpdateAsync(importString.RepeatOnce());", "\n break;", "\n case StreamImportOptions smio:", - "\n var keyedStreamImport = new KeyedStreamImport(Guid.NewGuid());", + "\n var importStream = new ImportStream(Guid.NewGuid());", "\n try", "\n {", - "\n keyedStreamImport = await (new KeyedStreamImport(smio, importVariable)).InitializeImportDataAsync(session) as KeyedStreamImport;", + "\n importStream = await (new ImportStream(smio, importVariable, session)).InitializeImportDataAsync() as ImportStream;", "\n }", "\n catch (Exception)", "\n {", "\n importSucceeded = false;", "\n }", - "\n activity = activity with {SourceId = keyedStreamImport.Id, ", + "\n activity = activity with {SourceId = importStream.Id, ", "\n Category = \"Import from Stream\"};", - "\n if (importSucceeded) await dataSource.UpdateAsync(keyedStreamImport.RepeatOnce());", + "\n if (importSucceeded) await dataSource.UpdateAsync(importStream.RepeatOnce());", "\n break;", "\n case DataSetImportOptions dsio:", - "\n var keyedDataSetImport = new KeyedDataSetImport(Guid.NewGuid());", + "\n var importDataSet = new ImportDataSet(Guid.NewGuid());", "\n try", "\n {", - "\n keyedDataSetImport = await (new KeyedDataSetImport(dsio, importVariable)).InitializeImportDataAsync(session) as KeyedDataSetImport;", + "\n importDataSet = await (new ImportDataSet(dsio, importVariable, session)).InitializeImportDataAsync() as ImportDataSet;", "\n }", "\n catch(Exception)", "\n {", "\n importSucceeded = false;", "\n }", - "\n activity = activity with {SourceId = keyedDataSetImport.Id, ", + "\n activity = activity with {SourceId = importDataSet.Id, ", "\n Category = \"Import from Data Set\"};", - "\n if (importSucceeded) await dataSource.UpdateAsync(keyedDataSetImport.RepeatOnce());", + "\n if (importSucceeded) await dataSource.UpdateAsync(importDataSet.RepeatOnce());", "\n break;", "\n default:", "\n throw new Exception(\"Import Options object is not an instance of an appropriate class.\");", "\n break;", "\n } ", - "\n await dataSource.UpdateAsync(activity.RepeatOnce());", + "\n await dataSource.UpdateAsync(activity.RepeatOnce());", "\n await dataSource.CommitAsync(); ", "\n return log;", "\n}" From 2af5815533eef8e7d23c5e5a35aabcf35ea86f04 Mon Sep 17 00:00:00 2001 From: akatz Date: Tue, 10 Jan 2023 16:25:15 +0100 Subject: [PATCH 25/37] Initialize export data, start --- ifrs17/Utils/ActivityLogsAndImports.ipynb | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/ifrs17/Utils/ActivityLogsAndImports.ipynb b/ifrs17/Utils/ActivityLogsAndImports.ipynb index 18171148..c6be78ea 100644 --- a/ifrs17/Utils/ActivityLogsAndImports.ipynb +++ b/ifrs17/Utils/ActivityLogsAndImports.ipynb @@ -36,6 +36,7 @@ "\nusing Systemorph.Vertex.DataSetReader;", "\nusing Systemorph.Vertex.Export.Excel.Builders;", "\nusing Systemorph.Vertex.Export;", + "\nusing Systemorph.Vertex.Export.Builders;", "\nusing Systemorph.Vertex.FileStorage;", "\n", "\npublic record StreamWrapper(Stream Stream, bool WillBeReused);", @@ -112,6 +113,40 @@ "execution_count": 0, "outputs": [] }, + { + "cell_type": "code", + "source": [ + "public record ExportFile : KeyedImportExport", + "\n{", + "\n protected DocumentBuilder Builder {get; set;}", + "\n", + "\n public string Name {get; init;}", + "\n", + "\n public string ContentType {get; init;}", + "\n", + "\n public ExportFile(DocumentBuilder builder, IDataSetImportVariable importVariable, ISessionVariable session)", + "\n {", + "\n Builder = builder;", + "\n DataSetReader = importVariable;", + "\n Session = session;", + "\n Id = Guid.NewGuid();", + "\n CreationTime = DateTime.UtcNow;", + "\n }", + "\n", + "\n public async Task InitializeExportDataAsync()", + "\n {", + "\n var mapping = await Builder.GetMappingAsync();", + "\n var storage = mapping.Storage as IFileReadStorage;", + "\n var stream = await storage.ReadAsync(mapping.FileName, Session.CancellationToken);", + "\n return this with {Name = Path.GetFileName(mapping.FileName), ", + "\n ContentType = Path.GetExtension(mapping.FileName)};", + "\n }", + "\n}" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, { "cell_type": "code", "source": [ From 3a3550e204c51c9158903fd4377734f0af82cceb Mon Sep 17 00:00:00 2001 From: akatz Date: Tue, 10 Jan 2023 16:42:12 +0100 Subject: [PATCH 26/37] first swipe of export data async --- ifrs17/Utils/ActivityLogsAndImports.ipynb | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/ifrs17/Utils/ActivityLogsAndImports.ipynb b/ifrs17/Utils/ActivityLogsAndImports.ipynb index c6be78ea..672cd44b 100644 --- a/ifrs17/Utils/ActivityLogsAndImports.ipynb +++ b/ifrs17/Utils/ActivityLogsAndImports.ipynb @@ -135,11 +135,26 @@ "\n", "\n public async Task InitializeExportDataAsync()", "\n {", + "\n byte[] content;", + "\n string format;", "\n var mapping = await Builder.GetMappingAsync();", "\n var storage = mapping.Storage as IFileReadStorage;", "\n var stream = await storage.ReadAsync(mapping.FileName, Session.CancellationToken);", + "\n using(MemoryStream ms = new MemoryStream())", + "\n {", + "\n await stream.CopyToAsync(ms);", + "\n content = ms.ToArray();", + "\n ms.Position = 0;", + "\n var dsRes = await DataSetReader.ReadFromStream(ms).ExecuteAsync();", + "\n format = dsRes.Format;", + "\n stream.Close();", + "\n await stream.DisposeAsync();", + "\n }", "\n return this with {Name = Path.GetFileName(mapping.FileName), ", - "\n ContentType = Path.GetExtension(mapping.FileName)};", + "\n ContentType = Path.GetExtension(mapping.FileName),", + "\n SerializedContent = content, ", + "\n Length = content == null ? null : (uint)content.Length,", + "\n Format = format ?? mapping.Format};", "\n }", "\n}" ], From a8a0fbf5a2200fe8fc8cf1fe0bf4bdb13a556433 Mon Sep 17 00:00:00 2001 From: akatz Date: Tue, 10 Jan 2023 16:46:44 +0100 Subject: [PATCH 27/37] add trivial constructor --- ifrs17/Utils/ActivityLogsAndImports.ipynb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ifrs17/Utils/ActivityLogsAndImports.ipynb b/ifrs17/Utils/ActivityLogsAndImports.ipynb index 672cd44b..94c92d46 100644 --- a/ifrs17/Utils/ActivityLogsAndImports.ipynb +++ b/ifrs17/Utils/ActivityLogsAndImports.ipynb @@ -133,6 +133,11 @@ "\n CreationTime = DateTime.UtcNow;", "\n }", "\n", + "\n public ExportFile(Guid id)", + "\n {", + "\n Id = id;", + "\n }", + "\n", "\n public async Task InitializeExportDataAsync()", "\n {", "\n byte[] content;", From c4912fba11949d5111bc6ced9535bcc1b1ea62b5 Mon Sep 17 00:00:00 2001 From: akatz Date: Tue, 10 Jan 2023 18:20:00 +0100 Subject: [PATCH 28/37] execute with store activity compiles --- ifrs17/Utils/ActivityLogsAndImports.ipynb | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/ifrs17/Utils/ActivityLogsAndImports.ipynb b/ifrs17/Utils/ActivityLogsAndImports.ipynb index 94c92d46..c8c248e2 100644 --- a/ifrs17/Utils/ActivityLogsAndImports.ipynb +++ b/ifrs17/Utils/ActivityLogsAndImports.ipynb @@ -38,6 +38,7 @@ "\nusing Systemorph.Vertex.Export;", "\nusing Systemorph.Vertex.Export.Builders;", "\nusing Systemorph.Vertex.FileStorage;", + "\nusing Systemorph.Vertex.Export.Builders.Interfaces;", "\n", "\npublic record StreamWrapper(Stream Stream, bool WillBeReused);", "\n", @@ -440,6 +441,28 @@ "execution_count": 0, "outputs": [] }, + { + "cell_type": "code", + "source": [ + "public static async Task ExecuteWithStoreActivityAsync(this IDocumentBuilder builder, ", + "\n ISessionVariable session, ", + "\n IDataSource dataSource, ", + "\n IDataSetImportVariable importVariable)", + "\n{", + "\n var exportResult = await builder.ExecuteAsync();", + "\n var exportFile = await (new ExportFile(builder as DocumentBuilder, importVariable, session)).InitializeExportDataAsync();", + "\n var activity = new ImportExportActivity(exportResult.ActivityLog, session) with {Category = \"Export to File\", ", + "\n SourceId = exportFile.Id};", + "\n await dataSource.UpdateAsync(exportFile.RepeatOnce());", + "\n await dataSource.UpdateAsync(activity.RepeatOnce());", + "\n await dataSource.CommitAsync();", + "\n return exportResult;", + "\n}" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, { "cell_type": "code", "source": [ From 9d0166c33696226e9da4e29db8ec7f984415f0ab Mon Sep 17 00:00:00 2001 From: akatz Date: Wed, 11 Jan 2023 09:21:45 +0100 Subject: [PATCH 29/37] Removed the buggy parts --- ifrs17/Utils/ActivityLogsAndImports.ipynb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ifrs17/Utils/ActivityLogsAndImports.ipynb b/ifrs17/Utils/ActivityLogsAndImports.ipynb index c8c248e2..4e98e2cf 100644 --- a/ifrs17/Utils/ActivityLogsAndImports.ipynb +++ b/ifrs17/Utils/ActivityLogsAndImports.ipynb @@ -142,7 +142,6 @@ "\n public async Task InitializeExportDataAsync()", "\n {", "\n byte[] content;", - "\n string format;", "\n var mapping = await Builder.GetMappingAsync();", "\n var storage = mapping.Storage as IFileReadStorage;", "\n var stream = await storage.ReadAsync(mapping.FileName, Session.CancellationToken);", @@ -150,9 +149,6 @@ "\n {", "\n await stream.CopyToAsync(ms);", "\n content = ms.ToArray();", - "\n ms.Position = 0;", - "\n var dsRes = await DataSetReader.ReadFromStream(ms).ExecuteAsync();", - "\n format = dsRes.Format;", "\n stream.Close();", "\n await stream.DisposeAsync();", "\n }", @@ -160,7 +156,7 @@ "\n ContentType = Path.GetExtension(mapping.FileName),", "\n SerializedContent = content, ", "\n Length = content == null ? null : (uint)content.Length,", - "\n Format = format ?? mapping.Format};", + "\n Format = mapping.Format};", "\n }", "\n}" ], From 619464794a9873d3db9e49390565c9b617df9311 Mon Sep 17 00:00:00 2001 From: akatz Date: Wed, 11 Jan 2023 11:13:26 +0100 Subject: [PATCH 30/37] update initialize --- PresentValueSeries/InitializeData.ipynb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/PresentValueSeries/InitializeData.ipynb b/PresentValueSeries/InitializeData.ipynb index bb01a0f9..691de565 100644 --- a/PresentValueSeries/InitializeData.ipynb +++ b/PresentValueSeries/InitializeData.ipynb @@ -67,7 +67,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"Dimensions.xlsx\").WithFormat(ImportFormats.AocConfiguration).WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"Dimensions.xlsx\")", + "\n .WithFormat(ImportFormats.AocConfiguration)", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -76,7 +79,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"DataNodes.xlsx\").WithFormat(\"DataNode\").WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"DataNodes.xlsx\")", + "\n .WithFormat(\"DataNode\")", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, From a9fd7383c52b89dc4c6a97d694f8a71c201d45af Mon Sep 17 00:00:00 2001 From: akatz Date: Wed, 11 Jan 2023 11:19:45 +0100 Subject: [PATCH 31/37] save the imports --- .../PresentValue - Episode 2.ipynb | 10 ++++-- .../PresentValue - Episode 3.ipynb | 35 +++++++++++++++---- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/PresentValueSeries/PresentValue - Episode 2.ipynb b/PresentValueSeries/PresentValue - Episode 2.ipynb index b4fd0906..11f600de 100644 --- a/PresentValueSeries/PresentValue - Episode 2.ipynb +++ b/PresentValueSeries/PresentValue - Episode 2.ipynb @@ -146,7 +146,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"YieldCurve.xlsx\").WithType().WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"YieldCurve.xlsx\")", + "\n .WithType()", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -155,7 +158,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"Cashflows.xlsx\").WithFormat(\"Cashflow\").WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"Cashflows.xlsx\")", + "\n .WithFormat(\"Cashflow\")", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, diff --git a/PresentValueSeries/PresentValue - Episode 3.ipynb b/PresentValueSeries/PresentValue - Episode 3.ipynb index 114e275b..9ab7f3f2 100644 --- a/PresentValueSeries/PresentValue - Episode 3.ipynb +++ b/PresentValueSeries/PresentValue - Episode 3.ipynb @@ -146,7 +146,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"DataNodes_CH.xlsx\").WithFormat(\"DataNode\").WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"DataNodes_CH.xlsx\")", + "\n .WithFormat(\"DataNode\")", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -155,7 +158,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"DataNodes_DE.xlsx\").WithFormat(\"DataNode\").WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"DataNodes_DE.xlsx\")", + "\n .WithFormat(\"DataNode\")", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -164,7 +170,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"YieldCurve.xlsx\").WithType().WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"YieldCurve.xlsx\")", + "\n .WithType()", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -173,7 +182,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"Cashflows.xlsx\").WithFormat(\"Cashflow\").WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"Cashflows.xlsx\")", + "\n .WithFormat(\"Cashflow\")", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -182,7 +194,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"CF_CH_2021_12.xlsx\").WithFormat(\"Cashflow\").WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"CF_CH_2021_12.xlsx\")", + "\n .WithFormat(\"Cashflow\")", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -191,7 +206,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"CF_DE_2021_12.xlsx\").WithFormat(\"Cashflow\").WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"CF_DE_2021_12.xlsx\")", + "\n .WithFormat(\"Cashflow\")", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -200,7 +218,10 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"CF_DE_2022_12.xlsx\").WithFormat(\"Cashflow\").WithTarget(DataSource).ExecuteAsync()" + "await Import.FromFile(\"CF_DE_2022_12.xlsx\")", + "\n .WithFormat(\"Cashflow\")", + "\n .WithTarget(DataSource)", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, From af636480780a0185d27fdd411a8d0134c936ad19 Mon Sep 17 00:00:00 2001 From: akatz Date: Wed, 11 Jan 2023 11:42:56 +0100 Subject: [PATCH 32/37] fix exports in map template --- ifrs17-template/Export/MapTemplate.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ifrs17-template/Export/MapTemplate.ipynb b/ifrs17-template/Export/MapTemplate.ipynb index 931953b7..a56e1ffe 100644 --- a/ifrs17-template/Export/MapTemplate.ipynb +++ b/ifrs17-template/Export/MapTemplate.ipynb @@ -155,7 +155,7 @@ "\n .GroupofContractConfiguration(typeof(ReinsurancePortfolio))", "\n .GroupofContractConfiguration(typeof(InsurancePortfolio))", "\n .MainTabConfigurationWoScenario(partition)", - "\n.ExecuteAsync()" + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -240,7 +240,7 @@ "\n .StateEnumConfiguration() ", "\n .DataNodeStateConfiguration(dataNodeStates)", "\n .MainTabConfigurationWoScenario(partition)", - "\n.ExecuteAsync()" + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -326,7 +326,7 @@ "\n .WithSource(Workspace)", "\n .DataNodeParameterConfiguration(dataNodeParameters)", "\n .MainTabConfiguration(partition)", - "\n.ExecuteAsync()" + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, From d64de56176bd01bb6a81f44bbfeab9e94bb3f5cc Mon Sep 17 00:00:00 2001 From: akatz Date: Wed, 11 Jan 2023 11:59:09 +0100 Subject: [PATCH 33/37] Use content to determine length --- ifrs17-template/Initialization/InitSystemorphToDatabase.ipynb | 2 +- ifrs17/Utils/ActivityLogsAndImports.ipynb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ifrs17-template/Initialization/InitSystemorphToDatabase.ipynb b/ifrs17-template/Initialization/InitSystemorphToDatabase.ipynb index 79af62d4..9d7288c4 100644 --- a/ifrs17-template/Initialization/InitSystemorphToDatabase.ipynb +++ b/ifrs17-template/Initialization/InitSystemorphToDatabase.ipynb @@ -96,7 +96,7 @@ "\n .WithType()", "\n .WithType()", "\n .WithTarget(DataSource)", - "\n .ExecuteAsync()" + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, diff --git a/ifrs17/Utils/ActivityLogsAndImports.ipynb b/ifrs17/Utils/ActivityLogsAndImports.ipynb index 4e98e2cf..2c1a471b 100644 --- a/ifrs17/Utils/ActivityLogsAndImports.ipynb +++ b/ifrs17/Utils/ActivityLogsAndImports.ipynb @@ -178,7 +178,7 @@ "\n return this with{CreationTime = DateTime.UtcNow, ", "\n Format = formatAndContent.Format ?? Options.Format,", "\n SerializedContent = formatAndContent.Content,", - "\n Length = SerializedContent != null ? (uint)SerializedContent.Length : null,", + "\n Length = formatAndContent.Content != null ? (uint)formatAndContent.Content.Length : null,", "\n };", "\n }", "\n", From 055f15ab284da910c4d545fe3c1a043a3dbaf748 Mon Sep 17 00:00:00 2001 From: akatz Date: Wed, 11 Jan 2023 12:02:00 +0100 Subject: [PATCH 34/37] Remove obsolete comment --- ifrs17/Utils/ActivityLogsAndImports.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ifrs17/Utils/ActivityLogsAndImports.ipynb b/ifrs17/Utils/ActivityLogsAndImports.ipynb index 2c1a471b..2ce8b659 100644 --- a/ifrs17/Utils/ActivityLogsAndImports.ipynb +++ b/ifrs17/Utils/ActivityLogsAndImports.ipynb @@ -72,8 +72,8 @@ "\n [Conversion(typeof(JsonConverter))]", "\n public string[] InfoMessages {get; init;}", "\n", - "\n public Guid SourceId {get; init;} // Check if converting to Guid causes issues. If not, leave Guid.", - "\n", + "\n public Guid SourceId {get; init;} ", + "\n ", "\n public ImportExportActivity(ActivityLog log, ISessionVariable session)", "\n {", "\n Id = Guid.NewGuid();", From 5e7e433cf60a5d5860b40fc49913079c5f19cac5 Mon Sep 17 00:00:00 2001 From: akatz Date: Wed, 11 Jan 2023 12:46:04 +0100 Subject: [PATCH 35/37] update to 1.6.4 version, which is functional now --- ifrs17/DataModel/DataStructure.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ifrs17/DataModel/DataStructure.ipynb b/ifrs17/DataModel/DataStructure.ipynb index 12db6b17..d1d0c2d2 100644 --- a/ifrs17/DataModel/DataStructure.ipynb +++ b/ifrs17/DataModel/DataStructure.ipynb @@ -59,10 +59,10 @@ "\n#r \"nuget:Systemorph.SharePoint,1.6.2\"", "\n#r \"nuget:Systemorph.OneDrive,1.6.2\"", "\n#r \"nuget:Systemorph.Scopes,1.6.2\"", - "\n#r \"nuget:Systemorph.Import,1.6.4-ci-20230110-092155\"", + "\n#r \"nuget:Systemorph.Import,1.6.4\"", "\n#r \"nuget:Systemorph.Test,1.6.2\"", - "\n#r \"nuget:Systemorph.Export,1.6.4-ci-20230110-092155\"", - "\n#r \"nuget:Systemorph.DataSetReader,1.6.4-ci-20230110-092155\"", + "\n#r \"nuget:Systemorph.Export,1.6.4\"", + "\n#r \"nuget:Systemorph.DataSetReader,1.6.4\"", "\n#r \"nuget:Systemorph.DataSource,1.6.0\"", "\n#r \"nuget:Systemorph.DataSource.Conversions,1.6.0\"", "\n#r \"nuget:Systemorph.Reporting,1.6.2\"", From 6899bdca553490991ec4f793c3d16099d3c65453 Mon Sep 17 00:00:00 2001 From: akatz Date: Thu, 12 Jan 2023 09:47:53 +0100 Subject: [PATCH 36/37] tests --- ifrs17-template/Test/MapTemplateAndImportTest.ipynb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ifrs17-template/Test/MapTemplateAndImportTest.ipynb b/ifrs17-template/Test/MapTemplateAndImportTest.ipynb index cd6603fd..95e7e89a 100644 --- a/ifrs17-template/Test/MapTemplateAndImportTest.ipynb +++ b/ifrs17-template/Test/MapTemplateAndImportTest.ipynb @@ -166,7 +166,7 @@ "\n .GroupofContractConfiguration(typeof(ReinsurancePortfolio))", "\n .GroupofContractConfiguration(typeof(InsurancePortfolio))", "\n .MainTabConfigurationWoScenario(partition)", - "\n.ExecuteAsync();", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader);", "\n", "\nexportResult.ActivityLog.Status.Should().Be(ActivityLogStatus.Succeeded);" ], @@ -184,7 +184,7 @@ "\n .GroupofContractConfiguration(typeof(ReinsurancePortfolio))", "\n .GroupofContractConfiguration(typeof(InsurancePortfolio))", "\n .MainTabConfigurationWoScenario(partition)", - "\n.ExecuteAsync()" + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -265,7 +265,7 @@ "\n .StateEnumConfiguration() ", "\n .DataNodeStateConfiguration(dataNodeStates)", "\n .MainTabConfigurationWoScenario(partition)", - "\n.ExecuteAsync();", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader);", "\n", "\nexportResult.ActivityLog.Status.Should().Be(ActivityLogStatus.Succeeded);" ], @@ -281,7 +281,7 @@ "\n .StateEnumConfiguration() ", "\n .DataNodeStateConfiguration(dataNodeStates)", "\n .MainTabConfigurationWoScenario(partition)", - "\n.ExecuteAsync()" + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -357,7 +357,7 @@ "\n .WithSource(Workspace)", "\n .DataNodeParameterConfiguration(dataNodeParameters)", "\n .MainTabConfiguration(partition)", - "\n.ExecuteAsync();", + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader);", "\n", "\nexportResult.ActivityLog.Status.Should().Be(ActivityLogStatus.Succeeded);" ], @@ -372,7 +372,7 @@ "\n .WithSource(Workspace)", "\n .DataNodeParameterConfiguration(dataNodeParameters)", "\n .MainTabConfiguration(partition)", - "\n.ExecuteAsync()" + "\n .ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, From 670f74f3769dfe1b9afe7363e8756c76f221e029 Mon Sep 17 00:00:00 2001 From: akatz Date: Thu, 12 Jan 2023 10:09:51 +0100 Subject: [PATCH 37/37] tests --- ifrs17-template/Test/SequenceImportTest.ipynb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ifrs17-template/Test/SequenceImportTest.ipynb b/ifrs17-template/Test/SequenceImportTest.ipynb index 8a1b0187..e5014953 100644 --- a/ifrs17-template/Test/SequenceImportTest.ipynb +++ b/ifrs17-template/Test/SequenceImportTest.ipynb @@ -55,7 +55,7 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/TransactionalData/NominalCashflows_CH_2020_12.csv\").WithFormat(ImportFormats.Cashflow).WithTarget(ws1).ExecuteAsync()" + "await Import.FromFile(\"../Files/TransactionalData/NominalCashflows_CH_2020_12.csv\").WithFormat(ImportFormats.Cashflow).WithTarget(ws1).ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -64,7 +64,7 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/TransactionalData/Actuals_CH_2020_12.csv\").WithFormat(ImportFormats.Actual).WithTarget(ws1).ExecuteAsync()" + "await Import.FromFile(\"../Files/TransactionalData/Actuals_CH_2020_12.csv\").WithFormat(ImportFormats.Actual).WithTarget(ws1).ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -112,7 +112,7 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/TransactionalData/Actuals_CH_2020_12.csv\").WithFormat(ImportFormats.Actual).WithTarget(ws2).ExecuteAsync()" + "await Import.FromFile(\"../Files/TransactionalData/Actuals_CH_2020_12.csv\").WithFormat(ImportFormats.Actual).WithTarget(ws2).ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0, @@ -121,7 +121,7 @@ { "cell_type": "code", "source": [ - "await Import.FromFile(\"../Files/TransactionalData/NominalCashflows_CH_2020_12.csv\").WithFormat(ImportFormats.Cashflow).WithTarget(ws2).ExecuteAsync()" + "await Import.FromFile(\"../Files/TransactionalData/NominalCashflows_CH_2020_12.csv\").WithFormat(ImportFormats.Cashflow).WithTarget(ws2).ExecuteWithStoreActivityAsync(Session, DataSource, DataSetReader)" ], "metadata": {}, "execution_count": 0,