From fa58882933f68e173c252bea09e41da05b95a735 Mon Sep 17 00:00:00 2001 From: Daniel Richter Date: Tue, 22 Oct 2024 11:24:05 +0200 Subject: [PATCH 01/12] Class for visualising the AST, output for terminal,file,png and file formatted. Class not complete --- .../memilio/generation/graph_visualization.py | 262 ++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100644 pycode/memilio-generation/memilio/generation/graph_visualization.py diff --git a/pycode/memilio-generation/memilio/generation/graph_visualization.py b/pycode/memilio-generation/memilio/generation/graph_visualization.py new file mode 100644 index 0000000000..d812446006 --- /dev/null +++ b/pycode/memilio-generation/memilio/generation/graph_visualization.py @@ -0,0 +1,262 @@ +############################################################################# +# Copyright (C) 2020-2024 MEmilio +# +# Authors: Daniel Richter +# +# Contact: Martin J. Kuehn +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################# + + +import os +import subprocess +from typing import Any, List, TextIO +from graphviz import Digraph +from clang.cindex import Cursor +import tempfile +import logging + +from memilio.generation import scanner, utility, intermediate_representation, scanner_config + + +class ASTViz: + + def __init__(self): + ast_cursor = Cursor() + self.node_counter = 0 + + def output_ast_terminal(self, ast_cursor: Cursor) -> None: + """ + Output the abstract syntax tree to terminal. + """ + _output_cursor_and_children_print(ast_cursor) + + def output_ast_file(self, ast_cursor: Cursor) -> None: + """ + Output the abstract syntax tree to file. + """ + with open('output_ast.txt', 'a') as f: + _output_cursor_and_children_file(ast_cursor, f) + print('AST written to ' + str(os.path.abspath(f.name))) + + def output_ast_png_all(self, ast_cursor: Cursor) -> None: + """ + Output the abstract syntax tree as a graph using Graphviz and save it to a file. + """ + + graph = Digraph(format='png') + + _output_cursor_and_children_graphviz_digraph(ast_cursor, graph) + + output_file = 'ast_graph' + + graph.render(output_file, view=False) + + output_path = os.path.abspath(f"{output_file}.png") + print(f'AST png written to {output_path}') + + def output_ast_digraph(self, ast_cursor: Cursor) -> None: + """ + Output the abstract syntax tree as a graph using Graphviz and save it to a file. + """ + + graph = Digraph(format='dot') + + _output_cursor_and_children_graphviz_digraph(ast_cursor, graph) + + output_file = 'ast_graph' + + graph.save(output_file + '.dot') + + # Ausgabe der Pfad-Informationen + output_path = os.path.abspath(f"{output_file}.dot") + print(f'AST digraph written to {output_path}') + + def output_ast_formatted(self, ast_cursor: Cursor) -> None: + + with open('output_ast_format.txt', 'w') as f: + _output_cursor_and_children_text(ast_cursor, f) + + print("AST format written to " + str(os.path.abspath(f.name))) + + +def indent2(level: int) -> str: + """Erstelle eine Einrückung basierend auf der Ebene.""" + return '│ ' * level + '├── ' + + +def _output_cursor_and_children_text(cursor: Cursor, f: TextIO, level: int = 0, node_counter: int = 0) -> None: + """ + Ausgabe des Cursors und seiner Kinder im Textformat, mit Hervorhebung für Ordner, Spelling und Kind-Typ. + + @param cursor: Der aktuelle Knoten des AST als Cursor-Objekt von libclang. + @param f: Offenes Dateiobjekt für die Ausgabe. + @param level: Die aktuelle Tiefe im AST für Einrückungszwecke. + """ + + node_counter += 1 + cursor_id = node_counter + # File-Pfad, Cursor-Art und spelling ermitteln + cursor_kind = f"" + file_path = cursor.location.file.name if cursor.location.file else "" + cursor_label = f'ID={cursor_id} {cursor.spelling} {cursor_kind} {file_path}' if cursor.spelling else f'{cursor_kind} {file_path}' + + if cursor.spelling: + cursor_label = (f'{cursor.spelling} ' + f'{cursor_kind} ' + f'{file_path}') + else: + cursor_label = f'{cursor_kind} [{file_path}]' + + # Schreibe das Label des aktuellen Cursors in die Datei + f.write(indent2(level) + cursor_label + '\n') + + # Rekursion für Kinder dieses Cursors + for child in cursor.get_children(): + _output_cursor_and_children_text(child, f, level + 1) + + +def _output_cursor_and_children_graphviz_digraph(cursor: Cursor, graph: Digraph, parent_node: str = None) -> None: + """ + Output the cursor and its children as a graph using Graphviz. + + @param cursor: The current node of the AST as a Cursor object from libclang. + @param graph: Graphviz Digraph object where the nodes and edges will be added. + @param parent_node: Name of the parent node in the graph (None for the root node). + """ + # Define a label for the current node in the graph + node_label = f"{cursor.kind.name}\n({cursor.spelling})" if cursor.spelling else cursor.kind.name + # Unique node ID using kind and hash + current_node = f"{cursor.kind.name}_{cursor.hash}" + + # Add the current node to the graph + graph.node(current_node, label=node_label) + + # If there is a parent node, create an edge from the parent to the current node + if parent_node: + graph.edge(parent_node, current_node) + + # Check if the cursor is a reference, and add a reference node if so + if cursor.kind.is_reference(): + referenced_label = f"ref_to_{cursor.referenced.kind.name}\n({cursor.referenced.spelling})" + referenced_node = f"ref_{cursor.referenced.hash}" + graph.node(referenced_node, label=referenced_label) + graph.edge(current_node, referenced_node) + + # Recurse for children of this cursor + for child in cursor.get_children(): + _output_cursor_and_children_graphviz_digraph( + child, graph, current_node) + + +def _output_cursor_and_children_file( + cursor: Cursor, f: TextIO, spaces: int = 0) -> None: + """ + Output this cursor and its children with minimal formatting to a file. + + @param cursor Represents the current node of the AST as an Cursor object from libclang. + @param f Open file object for output. + @param spaces Number of spaces. + """ + output_cursor_file(cursor, f, spaces) + if cursor.kind.is_reference(): + f.write(indent(spaces) + 'reference to:\n') + output_cursor_file(cursor.referenced, f, spaces+1) + + # Recurse for children of this cursor + has_children = False + for c in cursor.get_children(): + if not has_children: + f.write(indent(spaces) + '{\n') + has_children = True + _output_cursor_and_children_file(c, f, spaces+1) + + if has_children: + f.write(indent(spaces) + '}\n') + + +def output_cursor_file(cursor: Cursor, f: TextIO, spaces: int) -> None: + """ + Low level cursor output to a file. + + @param cursor Represents the current node of the AST as an Cursor object from libclang. + @param f Open file object for output. + @param spaces Number of spaces. + """ + spelling = '' + displayname = '' + + if cursor.spelling: + spelling = cursor.spelling + if cursor.displayname: + displayname = cursor.displayname + kind = cursor.kind + + f.write(indent(spaces) + spelling + ' <' + str(kind) + '> ') + if cursor.location.file: + f.write(cursor.location.file.name + '\n') + f.write(indent(spaces+1) + '"' + displayname + '"\n') + + +def indent(spaces: int) -> str: + """ + Indentation string for pretty-printing. + + @param spaces Number of spaces. + """ + return ' '*spaces + + +def _output_cursor_and_children_print(cursor: Cursor, spaces: int = 0) -> None: + """ + Output this cursor and its children with minimal formatting to the terminal. + + @param cursor Represents the current node of the AST as an Cursor object from libclang. + @param spaces [Default = 0] Number of spaces. + """ + _output_cursor_print(cursor, spaces) + if cursor.kind.is_reference(): + print(indent(spaces) + 'reference to:') + _output_cursor_print(cursor.referenced, spaces+1) + + # Recurse for children of this cursor + has_children = False + for c in cursor.get_children(): + if not has_children: + print(indent(spaces) + '{') + has_children = True + _output_cursor_and_children_print(c, spaces+1) + + if has_children: + print(indent(spaces) + '}') + + +def _output_cursor_print(cursor: Cursor, spaces: int) -> None: + """ + Low level cursor output to the terminal. + + @param cursor Represents the current node of the AST as an Cursor object from libclang. + @param spaces Number of spaces. + """ + spelling = '' + displayname = '' + + if cursor.spelling: + spelling = cursor.spelling + if cursor.displayname: + displayname = cursor.displayname + kind = cursor.kind + + print(indent(spaces) + spelling, '<' + str(kind) + '>') + print(indent(spaces+1) + '"' + displayname + '"') From f5637af43ae4a97af42e5a19ca823d5572cc6d90 Mon Sep 17 00:00:00 2001 From: Daniel Richter Date: Wed, 23 Oct 2024 14:24:17 +0200 Subject: [PATCH 02/12] scanner and graph_visualization, added output_ast_png_limit to output the ast as png with a starting node and max depth, added get_node_by_index to search a node based on its index in the cursor_nodes list. Not complete. --- .../memilio/generation/graph_visualization.py | 72 +++++++++++------- .../memilio/generation/scanner.py | 74 ++++++++++++++----- 2 files changed, 103 insertions(+), 43 deletions(-) diff --git a/pycode/memilio-generation/memilio/generation/graph_visualization.py b/pycode/memilio-generation/memilio/generation/graph_visualization.py index d812446006..7f48d0c700 100644 --- a/pycode/memilio-generation/memilio/generation/graph_visualization.py +++ b/pycode/memilio-generation/memilio/generation/graph_visualization.py @@ -38,52 +38,72 @@ def __init__(self): def output_ast_terminal(self, ast_cursor: Cursor) -> None: """ - Output the abstract syntax tree to terminal. + Output the abstract syntax tree to terminal. Not formatted. """ _output_cursor_and_children_print(ast_cursor) def output_ast_file(self, ast_cursor: Cursor) -> None: """ - Output the abstract syntax tree to file. + Output the abstract syntax tree to file. Not formatted. """ with open('output_ast.txt', 'a') as f: _output_cursor_and_children_file(ast_cursor, f) print('AST written to ' + str(os.path.abspath(f.name))) - def output_ast_png_all(self, ast_cursor: Cursor) -> None: + # def output_ast_png_all(self, ast_cursor: Cursor) -> None: + # """ + # Output the abstract syntax tree as a graph using Graphviz and save it to a file. + # """ + + # graph = Digraph(format='png') + + # _output_cursor_and_children_graphviz_digraph(ast_cursor, graph) + + # output_file = 'ast_graph' + + # graph.render(output_file, view=False) + + # output_path = os.path.abspath(f"{output_file}.png") + # print(f'AST png written to {output_path}') + + def output_ast_png_limit(self, ast_cursor: Cursor, max_depth: int, ) -> None: """ - Output the abstract syntax tree as a graph using Graphviz and save it to a file. + Output the abstract syntax tree to a .png. Set the starting node and the max depth. """ graph = Digraph(format='png') - _output_cursor_and_children_graphviz_digraph(ast_cursor, graph) + _output_cursor_and_children_graphviz_digraph( + ast_cursor, graph, max_depth, 0) - output_file = 'ast_graph' + output_file = 'ast_graph_png_limit' graph.render(output_file, view=False) output_path = os.path.abspath(f"{output_file}.png") print(f'AST png written to {output_path}') - def output_ast_digraph(self, ast_cursor: Cursor) -> None: - """ - Output the abstract syntax tree as a graph using Graphviz and save it to a file. - """ + # def output_ast_digraph(self, ast_cursor: Cursor) -> None: + # """ + # Output the abstract syntax tree as a graph using Graphviz and save it to a file. + # """ - graph = Digraph(format='dot') + # graph = Digraph(format='dot') - _output_cursor_and_children_graphviz_digraph(ast_cursor, graph) + # _output_cursor_and_children_graphviz_digraph(ast_cursor, graph) - output_file = 'ast_graph' + # output_file = 'ast_graph' - graph.save(output_file + '.dot') + # graph.save(output_file + '.dot') - # Ausgabe der Pfad-Informationen - output_path = os.path.abspath(f"{output_file}.dot") - print(f'AST digraph written to {output_path}') + # # Ausgabe der Pfad-Informationen + # output_path = os.path.abspath(f"{output_file}.dot") + # print(f'AST digraph written to {output_path}') def output_ast_formatted(self, ast_cursor: Cursor) -> None: + """ + Output the abstract syntax tree to a file. Formatted. + """ with open('output_ast_format.txt', 'w') as f: _output_cursor_and_children_text(ast_cursor, f) @@ -92,13 +112,15 @@ def output_ast_formatted(self, ast_cursor: Cursor) -> None: def indent2(level: int) -> str: - """Erstelle eine Einrückung basierend auf der Ebene.""" + """ + Create an indentation based on the level. + """ return '│ ' * level + '├── ' def _output_cursor_and_children_text(cursor: Cursor, f: TextIO, level: int = 0, node_counter: int = 0) -> None: """ - Ausgabe des Cursors und seiner Kinder im Textformat, mit Hervorhebung für Ordner, Spelling und Kind-Typ. + Output of the cursor and its children in text format, with highlighting for folder, spelling and child type. @param cursor: Der aktuelle Knoten des AST als Cursor-Objekt von libclang. @param f: Offenes Dateiobjekt für die Ausgabe. @@ -107,27 +129,25 @@ def _output_cursor_and_children_text(cursor: Cursor, f: TextIO, level: int = 0, node_counter += 1 cursor_id = node_counter - # File-Pfad, Cursor-Art und spelling ermitteln + cursor_kind = f"" file_path = cursor.location.file.name if cursor.location.file else "" cursor_label = f'ID={cursor_id} {cursor.spelling} {cursor_kind} {file_path}' if cursor.spelling else f'{cursor_kind} {file_path}' if cursor.spelling: - cursor_label = (f'{cursor.spelling} ' + cursor_label = (f'ID={cursor_id}{cursor.spelling} ' f'{cursor_kind} ' f'{file_path}') else: cursor_label = f'{cursor_kind} [{file_path}]' - # Schreibe das Label des aktuellen Cursors in die Datei f.write(indent2(level) + cursor_label + '\n') - # Rekursion für Kinder dieses Cursors for child in cursor.get_children(): _output_cursor_and_children_text(child, f, level + 1) -def _output_cursor_and_children_graphviz_digraph(cursor: Cursor, graph: Digraph, parent_node: str = None) -> None: +def _output_cursor_and_children_graphviz_digraph(cursor: Cursor, graph: Digraph, max_d: int, current_d: int, parent_node: str = None) -> None: """ Output the cursor and its children as a graph using Graphviz. @@ -135,6 +155,8 @@ def _output_cursor_and_children_graphviz_digraph(cursor: Cursor, graph: Digraph, @param graph: Graphviz Digraph object where the nodes and edges will be added. @param parent_node: Name of the parent node in the graph (None for the root node). """ + if current_d > max_d: + return # Define a label for the current node in the graph node_label = f"{cursor.kind.name}\n({cursor.spelling})" if cursor.spelling else cursor.kind.name # Unique node ID using kind and hash @@ -157,7 +179,7 @@ def _output_cursor_and_children_graphviz_digraph(cursor: Cursor, graph: Digraph, # Recurse for children of this cursor for child in cursor.get_children(): _output_cursor_and_children_graphviz_digraph( - child, graph, current_node) + child, graph, max_d, current_d + 1, current_node) def _output_cursor_and_children_file( diff --git a/pycode/memilio-generation/memilio/generation/scanner.py b/pycode/memilio-generation/memilio/generation/scanner.py index 2f14e646b3..76d465f58e 100755 --- a/pycode/memilio-generation/memilio/generation/scanner.py +++ b/pycode/memilio-generation/memilio/generation/scanner.py @@ -27,8 +27,11 @@ import subprocess import sys import tempfile +import logging +from graphviz import Digraph from typing import TYPE_CHECKING, Any, Callable from warnings import catch_warnings +from memilio.generation import graph_visualization from clang.cindex import * from typing_extensions import Self @@ -54,6 +57,8 @@ def __init__(self: Self, conf: ScannerConfig) -> None: utility.try_set_libclang_path( self.config.optional.get("libclang_library_path")) self.ast = None + self.node_counter = 0 + self.cursor_ids = [] self.create_ast() def create_ast(self: Self) -> None: @@ -73,16 +78,29 @@ def create_ast(self: Self) -> None: for command in commands: for argument in command.arguments: if (argument != '-Wno-unknown-warning' and - argument != "--driver-mode=g++" and argument != "-O3"): + argument != "--driver-mode=g++" and argument != "-O3" and argument != "-Werror" and argument != "-Wshadow"): file_args.append(argument) file_args = file_args[1:-4] + + # Removing only the first and last arguments could miss important flags. + # Safer approach is to include all relevant arguments except for optimization/debug flags + # file_args = file_args[1:-4] if len(file_args) > 5 else file_args + clang_cmd = [ "clang-14", self.config.source_file, "-std=c++17", '-emit-ast', '-o', '-'] clang_cmd.extend(file_args) - clang_cmd_result = subprocess.run(clang_cmd, stdout=subprocess.PIPE) - clang_cmd_result.check_returncode() + try: + clang_cmd_result = subprocess.run( + clang_cmd, stdout=subprocess.PIPE) + clang_cmd_result.check_returncode() + except subprocess.CalledProcessError as e: + # Capture standard error and output + logging.error( + f"Clang failed with return code {e.returncode}. Error: {clang_cmd_result.stderr.decode()}") + raise RuntimeError( + f"Clang AST generation failed. See error log for details.") # Since `clang.Index.read` expects a file path, write generated abstract syntax tree to a # temporary named file. This file will be automatically deleted when closed. @@ -90,6 +108,40 @@ def create_ast(self: Self) -> None: ast_file.write(clang_cmd_result.stdout) self.ast = idx.read(ast_file.name) + self.assing_ast_with_ids(self.ast.cursor) + + logging.info("AST generation completed successfully.") + + def assing_ast_with_ids(self, cursor: Cursor) -> None: + """ + Traverse the AST and assign a unique ID to each node during traversal. + + Parameters: + @cursor: The current node (Cursor) in the AST to traverse. + """ + + self.node_counter += 1 # Erhöhe den ID-Zähler + cursor_id = self.node_counter # Weise dem Knoten die aktuelle ID zu + + # Hier könntest du den Knoten nach Bedarf weiterverarbeiten. + # Zum Beispiel: Ausgabe des Knotens und seiner ID + logging.debug( + f"Node {cursor.spelling or cursor.kind} assigned ID {cursor_id}") + + # Rekursiv durch die Kinderknoten traversieren und IDs zuweisen + for child in cursor.get_children(): + self.assing_ast_with_ids(child) + + def get_node_id(self, cursor: Cursor) -> int: + """ + Gibt die ID des angegebenen Knotens zurück. + """ + + for c, id in self.cursor_ids: + if c == cursor: + return id + return 0 + def extract_results(self: Self) -> IntermediateRepresentation: """ Extract the information of the abstract syntax tree and save them in the dataclass intermed_repr. @@ -98,7 +150,7 @@ def extract_results(self: Self) -> IntermediateRepresentation: @return Information extracted from the model saved as an IntermediateRepresentation. """ intermed_repr = IntermediateRepresentation() - utility.output_cursor_print(self.ast.cursor, 1) + graph_visualization._output_cursor_print(self.ast.cursor, 1) self.find_node(self.ast.cursor, intermed_repr) # self.output_ast_file() self.finalize(intermed_repr) @@ -346,17 +398,3 @@ def finalize(self: Self, intermed_repr: IntermediateRepresentation) -> None: # check for missing data intermed_repr.check_complete_data(self.config.optional) - - def output_ast(self: Self) -> None: - """ - Output the abstract syntax tree to terminal. - """ - utility.output_cursor_and_children(self.ast.cursor) - - def output_ast_file(self: Self) -> None: - """ - Output the abstract syntax tree to file. - """ - with open('output_ast.txt', 'a') as f: - utility.output_cursor_and_children_file(self.ast.cursor, f) - print('AST written to ' + str(os.path.abspath(f.name))) From 973f64818eddb96e9bb8d38aa1b295f11532a4b4 Mon Sep 17 00:00:00 2001 From: Daniel Richter Date: Thu, 24 Oct 2024 10:27:45 +0200 Subject: [PATCH 03/12] Same as before --- .../memilio/generation/graph_visualization.py | 11 ++++++----- .../memilio/generation/scanner.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/pycode/memilio-generation/memilio/generation/graph_visualization.py b/pycode/memilio-generation/memilio/generation/graph_visualization.py index 7f48d0c700..0123bb2b75 100644 --- a/pycode/memilio-generation/memilio/generation/graph_visualization.py +++ b/pycode/memilio-generation/memilio/generation/graph_visualization.py @@ -48,7 +48,7 @@ def output_ast_file(self, ast_cursor: Cursor) -> None: """ with open('output_ast.txt', 'a') as f: _output_cursor_and_children_file(ast_cursor, f) - print('AST written to ' + str(os.path.abspath(f.name))) + print('AST-file written to ' + str(os.path.abspath(f.name))) # def output_ast_png_all(self, ast_cursor: Cursor) -> None: # """ @@ -81,7 +81,7 @@ def output_ast_png_limit(self, ast_cursor: Cursor, max_depth: int, ) -> None: graph.render(output_file, view=False) output_path = os.path.abspath(f"{output_file}.png") - print(f'AST png written to {output_path}') + print(f'AST-png written to {output_path}') # def output_ast_digraph(self, ast_cursor: Cursor) -> None: # """ @@ -247,21 +247,22 @@ def _output_cursor_and_children_print(cursor: Cursor, spaces: int = 0) -> None: @param cursor Represents the current node of the AST as an Cursor object from libclang. @param spaces [Default = 0] Number of spaces. """ + _output_cursor_print(cursor, spaces) if cursor.kind.is_reference(): - print(indent(spaces) + 'reference to:') + print(indent2(spaces) + 'reference to:') _output_cursor_print(cursor.referenced, spaces+1) # Recurse for children of this cursor has_children = False for c in cursor.get_children(): if not has_children: - print(indent(spaces) + '{') + print(indent2(spaces) + '{') has_children = True _output_cursor_and_children_print(c, spaces+1) if has_children: - print(indent(spaces) + '}') + print(indent2(spaces) + '}') def _output_cursor_print(cursor: Cursor, spaces: int) -> None: diff --git a/pycode/memilio-generation/memilio/generation/scanner.py b/pycode/memilio-generation/memilio/generation/scanner.py index 76d465f58e..ef346834d2 100755 --- a/pycode/memilio-generation/memilio/generation/scanner.py +++ b/pycode/memilio-generation/memilio/generation/scanner.py @@ -59,6 +59,7 @@ def __init__(self: Self, conf: ScannerConfig) -> None: self.ast = None self.node_counter = 0 self.cursor_ids = [] + self.cursor_nodes = [] self.create_ast() def create_ast(self: Self) -> None: @@ -123,6 +124,9 @@ def assing_ast_with_ids(self, cursor: Cursor) -> None: self.node_counter += 1 # Erhöhe den ID-Zähler cursor_id = self.node_counter # Weise dem Knoten die aktuelle ID zu + self.cursor_ids.append((cursor, cursor_id)) + self.cursor_nodes.append(cursor) + # Hier könntest du den Knoten nach Bedarf weiterverarbeiten. # Zum Beispiel: Ausgabe des Knotens und seiner ID logging.debug( @@ -142,6 +146,15 @@ def get_node_id(self, cursor: Cursor) -> int: return id return 0 + def get_node_by_index(self, index: int) -> Cursor: + """ + Gibt den Knoten an der angegebenen Indexposition zurück. + Wenn der Index außerhalb der Grenzen liegt, wird None zurückgegeben. + """ + if 0 <= index < len(self.cursor_nodes): + return self.cursor_nodes[index] + return None + def extract_results(self: Self) -> IntermediateRepresentation: """ Extract the information of the abstract syntax tree and save them in the dataclass intermed_repr. From 10b5b6e313ff25a8e0a9c4d412be2991c3a3d108 Mon Sep 17 00:00:00 2001 From: Daniel Richter Date: Fri, 25 Oct 2024 11:06:51 +0200 Subject: [PATCH 04/12] Changes in graph_visualization and scanner, finalized the class ASTViz. Final commit. --- .../memilio/generation/graph_visualization.py | 231 +++++------------- .../memilio/generation/scanner.py | 25 +- pycode/memilio-generation/setup.py | 2 +- 3 files changed, 69 insertions(+), 189 deletions(-) diff --git a/pycode/memilio-generation/memilio/generation/graph_visualization.py b/pycode/memilio-generation/memilio/generation/graph_visualization.py index 0123bb2b75..758b9d8ad2 100644 --- a/pycode/memilio-generation/memilio/generation/graph_visualization.py +++ b/pycode/memilio-generation/memilio/generation/graph_visualization.py @@ -18,55 +18,30 @@ # limitations under the License. ############################################################################# - import os -import subprocess -from typing import Any, List, TextIO +from typing import TextIO from graphviz import Digraph from clang.cindex import Cursor -import tempfile -import logging - -from memilio.generation import scanner, utility, intermediate_representation, scanner_config +from memilio.generation import Scanner class ASTViz: + """ + Class for plotting the abstract-syntax-tree in different formats + """ - def __init__(self): - ast_cursor = Cursor() - self.node_counter = 0 - - def output_ast_terminal(self, ast_cursor: Cursor) -> None: + @staticmethod + def output_ast_terminal(ast_cursor: Cursor) -> None: """ Output the abstract syntax tree to terminal. Not formatted. """ - _output_cursor_and_children_print(ast_cursor) - - def output_ast_file(self, ast_cursor: Cursor) -> None: - """ - Output the abstract syntax tree to file. Not formatted. - """ - with open('output_ast.txt', 'a') as f: - _output_cursor_and_children_file(ast_cursor, f) - print('AST-file written to ' + str(os.path.abspath(f.name))) - # def output_ast_png_all(self, ast_cursor: Cursor) -> None: - # """ - # Output the abstract syntax tree as a graph using Graphviz and save it to a file. - # """ - - # graph = Digraph(format='png') - - # _output_cursor_and_children_graphviz_digraph(ast_cursor, graph) - - # output_file = 'ast_graph' - - # graph.render(output_file, view=False) + _output_cursor_and_children_print(ast_cursor) - # output_path = os.path.abspath(f"{output_file}.png") - # print(f'AST png written to {output_path}') + print(f'AST-terminal written.') - def output_ast_png_limit(self, ast_cursor: Cursor, max_depth: int, ) -> None: + @staticmethod + def output_ast_png(ast_cursor: Cursor, max_depth: int, ) -> None: """ Output the abstract syntax tree to a .png. Set the starting node and the max depth. """ @@ -83,24 +58,8 @@ def output_ast_png_limit(self, ast_cursor: Cursor, max_depth: int, ) -> None: output_path = os.path.abspath(f"{output_file}.png") print(f'AST-png written to {output_path}') - # def output_ast_digraph(self, ast_cursor: Cursor) -> None: - # """ - # Output the abstract syntax tree as a graph using Graphviz and save it to a file. - # """ - - # graph = Digraph(format='dot') - - # _output_cursor_and_children_graphviz_digraph(ast_cursor, graph) - - # output_file = 'ast_graph' - - # graph.save(output_file + '.dot') - - # # Ausgabe der Pfad-Informationen - # output_path = os.path.abspath(f"{output_file}.dot") - # print(f'AST digraph written to {output_path}') - - def output_ast_formatted(self, ast_cursor: Cursor) -> None: + @staticmethod + def output_ast_formatted(ast_cursor: Cursor) -> None: """ Output the abstract syntax tree to a file. Formatted. """ @@ -108,7 +67,7 @@ def output_ast_formatted(self, ast_cursor: Cursor) -> None: with open('output_ast_format.txt', 'w') as f: _output_cursor_and_children_text(ast_cursor, f) - print("AST format written to " + str(os.path.abspath(f.name))) + print("AST-format written to " + str(os.path.abspath(f.name))) def indent2(level: int) -> str: @@ -118,33 +77,62 @@ def indent2(level: int) -> str: return '│ ' * level + '├── ' -def _output_cursor_and_children_text(cursor: Cursor, f: TextIO, level: int = 0, node_counter: int = 0) -> None: +def _output_cursor_and_children_text(cursor: Cursor, f: TextIO, level: int = 0, cursor_id: int = 0) -> int: """ Output of the cursor and its children in text format, with highlighting for folder, spelling and child type. - @param cursor: Der aktuelle Knoten des AST als Cursor-Objekt von libclang. - @param f: Offenes Dateiobjekt für die Ausgabe. - @param level: Die aktuelle Tiefe im AST für Einrückungszwecke. + @param cursor: The current node of the AST as a cursor object from libclang. + @param f: Open file object for output. + @param level: The current depth in the AST for indentation purposes. + @param cursor_id: A unique ID to identify each cursor. """ - node_counter += 1 - cursor_id = node_counter - cursor_kind = f"" file_path = cursor.location.file.name if cursor.location.file else "" - cursor_label = f'ID={cursor_id} {cursor.spelling} {cursor_kind} {file_path}' if cursor.spelling else f'{cursor_kind} {file_path}' if cursor.spelling: - cursor_label = (f'ID={cursor_id}{cursor.spelling} ' + cursor_label = (f'ID={cursor_id} {cursor.spelling} ' f'{cursor_kind} ' f'{file_path}') else: - cursor_label = f'{cursor_kind} [{file_path}]' + cursor_label = f'ID={cursor_id} {cursor_kind} [{file_path}]' f.write(indent2(level) + cursor_label + '\n') for child in cursor.get_children(): - _output_cursor_and_children_text(child, f, level + 1) + cursor_id = _output_cursor_and_children_text( + child, f, level + 1, cursor_id + 1) + return cursor_id + + +def _output_cursor_and_children_print(cursor: Cursor, level: int = 0, cursor_id: int = 0) -> int: + """ + Prints the current cursor and its child elements in text format, + with highlighting for folder, case and node type. + + @param cursor: The current node of the AST as a libclang cursor object. + @param f: Open file object for output. + @param level: The current depth in the AST for indentation purposes. + @param cursor_id: A unique ID to identify each cursor. + @return: The current cursor ID for subsequent nodes. + """ + + cursor_id = Scanner.get_node_id(cursor) - 1 + + cursor_kind = f"" + file_path = cursor.location.file.name if cursor.location.file else "" + cursor_label = ( + f'ID={cursor_id} {cursor.spelling} {cursor_kind} {file_path}' + if cursor.spelling else + f'ID={cursor_id} {cursor_kind} {file_path}' + ) + + print(indent2(level) + cursor_label) + + for child in cursor.get_children(): + cursor_id = _output_cursor_and_children_print( + child, level + 1, cursor_id + 1) + return cursor_id def _output_cursor_and_children_graphviz_digraph(cursor: Cursor, graph: Digraph, max_d: int, current_d: int, parent_node: str = None) -> None: @@ -157,129 +145,22 @@ def _output_cursor_and_children_graphviz_digraph(cursor: Cursor, graph: Digraph, """ if current_d > max_d: return - # Define a label for the current node in the graph + node_label = f"{cursor.kind.name}\n({cursor.spelling})" if cursor.spelling else cursor.kind.name - # Unique node ID using kind and hash + current_node = f"{cursor.kind.name}_{cursor.hash}" - # Add the current node to the graph graph.node(current_node, label=node_label) - # If there is a parent node, create an edge from the parent to the current node if parent_node: graph.edge(parent_node, current_node) - # Check if the cursor is a reference, and add a reference node if so if cursor.kind.is_reference(): referenced_label = f"ref_to_{cursor.referenced.kind.name}\n({cursor.referenced.spelling})" referenced_node = f"ref_{cursor.referenced.hash}" graph.node(referenced_node, label=referenced_label) graph.edge(current_node, referenced_node) - # Recurse for children of this cursor for child in cursor.get_children(): _output_cursor_and_children_graphviz_digraph( child, graph, max_d, current_d + 1, current_node) - - -def _output_cursor_and_children_file( - cursor: Cursor, f: TextIO, spaces: int = 0) -> None: - """ - Output this cursor and its children with minimal formatting to a file. - - @param cursor Represents the current node of the AST as an Cursor object from libclang. - @param f Open file object for output. - @param spaces Number of spaces. - """ - output_cursor_file(cursor, f, spaces) - if cursor.kind.is_reference(): - f.write(indent(spaces) + 'reference to:\n') - output_cursor_file(cursor.referenced, f, spaces+1) - - # Recurse for children of this cursor - has_children = False - for c in cursor.get_children(): - if not has_children: - f.write(indent(spaces) + '{\n') - has_children = True - _output_cursor_and_children_file(c, f, spaces+1) - - if has_children: - f.write(indent(spaces) + '}\n') - - -def output_cursor_file(cursor: Cursor, f: TextIO, spaces: int) -> None: - """ - Low level cursor output to a file. - - @param cursor Represents the current node of the AST as an Cursor object from libclang. - @param f Open file object for output. - @param spaces Number of spaces. - """ - spelling = '' - displayname = '' - - if cursor.spelling: - spelling = cursor.spelling - if cursor.displayname: - displayname = cursor.displayname - kind = cursor.kind - - f.write(indent(spaces) + spelling + ' <' + str(kind) + '> ') - if cursor.location.file: - f.write(cursor.location.file.name + '\n') - f.write(indent(spaces+1) + '"' + displayname + '"\n') - - -def indent(spaces: int) -> str: - """ - Indentation string for pretty-printing. - - @param spaces Number of spaces. - """ - return ' '*spaces - - -def _output_cursor_and_children_print(cursor: Cursor, spaces: int = 0) -> None: - """ - Output this cursor and its children with minimal formatting to the terminal. - - @param cursor Represents the current node of the AST as an Cursor object from libclang. - @param spaces [Default = 0] Number of spaces. - """ - - _output_cursor_print(cursor, spaces) - if cursor.kind.is_reference(): - print(indent2(spaces) + 'reference to:') - _output_cursor_print(cursor.referenced, spaces+1) - - # Recurse for children of this cursor - has_children = False - for c in cursor.get_children(): - if not has_children: - print(indent2(spaces) + '{') - has_children = True - _output_cursor_and_children_print(c, spaces+1) - - if has_children: - print(indent2(spaces) + '}') - - -def _output_cursor_print(cursor: Cursor, spaces: int) -> None: - """ - Low level cursor output to the terminal. - - @param cursor Represents the current node of the AST as an Cursor object from libclang. - @param spaces Number of spaces. - """ - spelling = '' - displayname = '' - - if cursor.spelling: - spelling = cursor.spelling - if cursor.displayname: - displayname = cursor.displayname - kind = cursor.kind - - print(indent(spaces) + spelling, '<' + str(kind) + '>') - print(indent(spaces+1) + '"' + displayname + '"') diff --git a/pycode/memilio-generation/memilio/generation/scanner.py b/pycode/memilio-generation/memilio/generation/scanner.py index ef346834d2..5652cf96b1 100755 --- a/pycode/memilio-generation/memilio/generation/scanner.py +++ b/pycode/memilio-generation/memilio/generation/scanner.py @@ -31,7 +31,7 @@ from graphviz import Digraph from typing import TYPE_CHECKING, Any, Callable from warnings import catch_warnings -from memilio.generation import graph_visualization + from clang.cindex import * from typing_extensions import Self @@ -46,6 +46,7 @@ class Scanner: """ Analyze the model and extract the needed information. """ + cursor_ids = [] def __init__(self: Self, conf: ScannerConfig) -> None: """ @@ -58,7 +59,7 @@ def __init__(self: Self, conf: ScannerConfig) -> None: self.config.optional.get("libclang_library_path")) self.ast = None self.node_counter = 0 - self.cursor_ids = [] + self.cursor_nodes = [] self.create_ast() @@ -121,35 +122,33 @@ def assing_ast_with_ids(self, cursor: Cursor) -> None: @cursor: The current node (Cursor) in the AST to traverse. """ - self.node_counter += 1 # Erhöhe den ID-Zähler - cursor_id = self.node_counter # Weise dem Knoten die aktuelle ID zu + self.node_counter += 1 + cursor_id = self.node_counter self.cursor_ids.append((cursor, cursor_id)) self.cursor_nodes.append(cursor) - # Hier könntest du den Knoten nach Bedarf weiterverarbeiten. - # Zum Beispiel: Ausgabe des Knotens und seiner ID logging.debug( f"Node {cursor.spelling or cursor.kind} assigned ID {cursor_id}") - # Rekursiv durch die Kinderknoten traversieren und IDs zuweisen for child in cursor.get_children(): self.assing_ast_with_ids(child) - def get_node_id(self, cursor: Cursor) -> int: + @staticmethod + def get_node_id(cursor: Cursor) -> int: """ - Gibt die ID des angegebenen Knotens zurück. + Returns the id of the current node. """ - for c, id in self.cursor_ids: + for c, id in Scanner.cursor_ids: if c == cursor: return id return 0 def get_node_by_index(self, index: int) -> Cursor: """ - Gibt den Knoten an der angegebenen Indexposition zurück. - Wenn der Index außerhalb der Grenzen liegt, wird None zurückgegeben. + Returns the node at the specified index position. + If the index is out of bounds, None is returned. """ if 0 <= index < len(self.cursor_nodes): return self.cursor_nodes[index] @@ -163,7 +162,7 @@ def extract_results(self: Self) -> IntermediateRepresentation: @return Information extracted from the model saved as an IntermediateRepresentation. """ intermed_repr = IntermediateRepresentation() - graph_visualization._output_cursor_print(self.ast.cursor, 1) + # graph_visualization._output_cursor_print(self.ast.cursor, 1) self.find_node(self.ast.cursor, intermed_repr) # self.output_ast_file() self.finalize(intermed_repr) diff --git a/pycode/memilio-generation/setup.py b/pycode/memilio-generation/setup.py index c415447a2e..1cc8cd362a 100644 --- a/pycode/memilio-generation/setup.py +++ b/pycode/memilio-generation/setup.py @@ -27,7 +27,7 @@ where=os.path.dirname(os.path.abspath(__file__))), setup_requires=['cmake'], install_requires=['libclang==14.0.6', - 'dataclasses', 'dataclasses_json', 'importlib-resources>=1.1.0; python_version < \'3.9\''], + 'dataclasses', 'dataclasses_json', 'graphviz', 'importlib-resources>=1.1.0; python_version < \'3.9\''], extras_require={'dev': []}, long_description='', test_suite='memilio.generation_test', From 80be1580bc165500b2d8b4e3ac3b2566f99e655e Mon Sep 17 00:00:00 2001 From: Daniel Richter Date: Fri, 25 Oct 2024 14:49:13 +0200 Subject: [PATCH 05/12] Error handling --- pycode/memilio-generation/memilio/generation/scanner.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pycode/memilio-generation/memilio/generation/scanner.py b/pycode/memilio-generation/memilio/generation/scanner.py index 5652cf96b1..785fe4f3f1 100755 --- a/pycode/memilio-generation/memilio/generation/scanner.py +++ b/pycode/memilio-generation/memilio/generation/scanner.py @@ -143,7 +143,7 @@ def get_node_id(cursor: Cursor) -> int: for c, id in Scanner.cursor_ids: if c == cursor: return id - return 0 + raise IndexError(f"Cursor {cursor} is out of bounds.") def get_node_by_index(self, index: int) -> Cursor: """ @@ -152,7 +152,8 @@ def get_node_by_index(self, index: int) -> Cursor: """ if 0 <= index < len(self.cursor_nodes): return self.cursor_nodes[index] - return None + raise IndexError( + f"Index {index} is out of bounds (0 to {len(self.cursor_nodes) - 1}).") def extract_results(self: Self) -> IntermediateRepresentation: """ From bd2ee9bd52a9cfa1e6c888ff44040e6cc4338649 Mon Sep 17 00:00:00 2001 From: Daniel Richter Date: Mon, 4 Nov 2024 09:43:09 +0100 Subject: [PATCH 06/12] Interpolate simulation results added. ast_handler with class for AST added. Changes in graph_visualization, scanner and utility. --- .../memilio/generation/ast_handler.py | 148 ++++++++++++++++++ .../memilio/generation/graph_visualization.py | 95 ++++++----- .../memilio/generation/scanner.py | 104 +----------- .../generation/template/template_cpp.txt | 14 ++ .../memilio/generation/utility.py | 102 ------------ 5 files changed, 223 insertions(+), 240 deletions(-) create mode 100644 pycode/memilio-generation/memilio/generation/ast_handler.py diff --git a/pycode/memilio-generation/memilio/generation/ast_handler.py b/pycode/memilio-generation/memilio/generation/ast_handler.py new file mode 100644 index 0000000000..1d3d4b5617 --- /dev/null +++ b/pycode/memilio-generation/memilio/generation/ast_handler.py @@ -0,0 +1,148 @@ +############################################################################# +# Copyright (C) 2020-2024 MEmilio +# +# Authors: Daniel Richter +# +# Contact: Martin J. Kuehn +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################# + +import os +import subprocess +import tempfile +import logging +from clang.cindex import Cursor, TranslationUnit, Index, CompilationDatabase +from typing import TYPE_CHECKING +from memilio.generation import utility + + +if TYPE_CHECKING: + from memilio.generation import ScannerConfig + +from typing_extensions import Self + + +class AST: + + def __init__(self: Self, conf: "ScannerConfig") -> None: + self.config = conf + self.cursor_id = -1 + self.id_to_val = dict() + self.val_to_id = dict() + self.cursor = None + self.translation_unit = self.create_ast() + + def create_ast(self: Self) -> TranslationUnit: + """ + Create an abstract syntax tree for the main model.cpp file with a corresponding CompilationDatabase. + A compile_commands.json is required (automatically generated in the build process). + """ + idx = Index.create() + + # Create the cmd arguments + file_args = [] + + dirname = utility.try_get_compilation_database_path( + self.config.skbuild_path_to_database) + compdb = CompilationDatabase.fromDirectory(dirname) + commands = compdb.getCompileCommands(self.config.source_file) + for command in commands: + for argument in command.arguments: + if (argument != '-Wno-unknown-warning' and + argument != "--driver-mode=g++" and argument != "-O3" and argument != "-Werror" and argument != "-Wshadow"): + file_args.append(argument) + file_args = file_args[1:-4] + + # Removing only the first and last arguments could miss important flags. + # Safer approach is to include all relevant arguments except for optimization/debug flags + # file_args = file_args[1:-4] if len(file_args) > 5 else file_args + + clang_cmd = [ + "clang-14", self.config.source_file, + "-std=c++17", '-emit-ast', '-o', '-'] + clang_cmd.extend(file_args) + + try: + clang_cmd_result = subprocess.run( + clang_cmd, stdout=subprocess.PIPE) + clang_cmd_result.check_returncode() + except subprocess.CalledProcessError as e: + # Capture standard error and output + logging.error( + f"Clang failed with return code {e.returncode}. Error: {clang_cmd_result.stderr.decode()}") + raise RuntimeError( + f"Clang AST generation failed. See error log for details.") + + # Since `clang.Index.read` expects a file path, write generated abstract syntax tree to a + # temporary named file. This file will be automatically deleted when closed. + with tempfile.NamedTemporaryFile() as ast_file: + ast_file.write(clang_cmd_result.stdout) + translation_unit = idx.read(ast_file.name) + + self.assing_ast_with_ids(translation_unit.cursor) + + logging.info("AST generation completed successfully.") + + return translation_unit + + def assing_ast_with_ids(self, cursor: Cursor) -> None: + """ + Traverse the AST and assign a unique ID to each node during traversal. + + @param cursor: The current node (Cursor) in the AST to traverse. + """ + # assing_ids umschreiben -> mapping + self.cursor_id += 1 + id = self.cursor_id + self.id_to_val[id] = cursor + + if cursor.hash in self.val_to_id.keys(): + self.val_to_id[cursor.hash].append(id) + else: + self.val_to_id[cursor.hash] = [id] + + logging.info( + f"Node {cursor.spelling or cursor.kind} assigned ID {id}") + + for child in cursor.get_children(): + self.assing_ast_with_ids(child) + + @property + def root_cursor(self): + return self.translation_unit.cursor + + def get_node_id(self, cursor: Cursor) -> int: + """ + Returns the id of the current node. + Extracs the key from the current cursor from the dictonary id_to_val + + @param cursor: The current node of the AST as a cursor object from libclang. + """ + for cursor_id in self.val_to_id[cursor.hash]: + + if self.id_to_val[cursor_id] == cursor: + + return cursor_id + raise IndexError(f"Cursor {cursor} is out of bounds.") + + def get_node_by_index(self, index: int) -> Cursor: + """ + Returns the node at the specified index position. + + @param index: Node_id from the ast. + """ + + if index < 0 or index >= len(self.id_to_val): + raise IndexError(f"Index {index} is out of bounds.") + return self.id_to_val[index] diff --git a/pycode/memilio-generation/memilio/generation/graph_visualization.py b/pycode/memilio-generation/memilio/generation/graph_visualization.py index 758b9d8ad2..19a7e98fc9 100644 --- a/pycode/memilio-generation/memilio/generation/graph_visualization.py +++ b/pycode/memilio-generation/memilio/generation/graph_visualization.py @@ -22,36 +22,41 @@ from typing import TextIO from graphviz import Digraph from clang.cindex import Cursor -from memilio.generation import Scanner +from .ast_handler import AST -class ASTViz: +class Visualization: """ - Class for plotting the abstract-syntax-tree in different formats + Class for plotting the abstract-syntax-tree in different formats. """ - @staticmethod - def output_ast_terminal(ast_cursor: Cursor) -> None: + def output_ast_terminal(ast: AST, cursor: Cursor) -> None: """ - Output the abstract syntax tree to terminal. Not formatted. + Output the abstract syntax tree to terminal. + + @param ast: ast object from AST class. + @param cursor: The current node of the AST as a cursor object from libclang. """ - _output_cursor_and_children_print(ast_cursor) + _output_cursor_and_children_print(cursor, ast) print(f'AST-terminal written.') @staticmethod - def output_ast_png(ast_cursor: Cursor, max_depth: int, ) -> None: + def output_ast_png(cursor: Cursor, max_depth: int, ) -> None: """ Output the abstract syntax tree to a .png. Set the starting node and the max depth. + + @param cursor: The current node of the AST as a cursor object from libclang. + @param max_depth: Maximal depth the graph displays. """ graph = Digraph(format='png') _output_cursor_and_children_graphviz_digraph( - ast_cursor, graph, max_depth, 0) + cursor, graph, max_depth, 0) - output_file = 'ast_graph_png_limit' + output_file = 'ast_graph' graph.render(output_file, view=False) @@ -59,80 +64,93 @@ def output_ast_png(ast_cursor: Cursor, max_depth: int, ) -> None: print(f'AST-png written to {output_path}') @staticmethod - def output_ast_formatted(ast_cursor: Cursor) -> None: + def output_ast_formatted(ast: AST, cursor: Cursor) -> None: """ - Output the abstract syntax tree to a file. Formatted. + Output the abstract syntax tree to a file. + + @param ast: ast object from AST class. + @param cursor: The current node of the AST as a cursor object from libclang. """ - with open('output_ast_format.txt', 'w') as f: - _output_cursor_and_children_text(ast_cursor, f) + with open('ast_formated.txt', 'w') as f: + _output_cursor_and_children_text(cursor, ast, f) - print("AST-format written to " + str(os.path.abspath(f.name))) + print("AST-formated written to " + str(os.path.abspath(f.name))) -def indent2(level: int) -> str: +def indent(level: int) -> str: """ Create an indentation based on the level. """ return '│ ' * level + '├── ' -def _output_cursor_and_children_text(cursor: Cursor, f: TextIO, level: int = 0, cursor_id: int = 0) -> int: +def newline() -> str: + """ + Create a new line. + """ + return '\n' + + +def _output_cursor_and_children_text(cursor: Cursor, ast: AST, f: TextIO, level: int = 0) -> int: """ Output of the cursor and its children in text format, with highlighting for folder, spelling and child type. @param cursor: The current node of the AST as a cursor object from libclang. + @param ast: AST from ast_handler.py @param f: Open file object for output. @param level: The current depth in the AST for indentation purposes. @param cursor_id: A unique ID to identify each cursor. """ + cursor_id = ast.get_node_id(cursor) + cursor_kind = f"" file_path = cursor.location.file.name if cursor.location.file else "" if cursor.spelling: - cursor_label = (f'ID={cursor_id} {cursor.spelling} ' + cursor_label = (f'ID:{cursor_id} {cursor.spelling} ' f'{cursor_kind} ' f'{file_path}') else: - cursor_label = f'ID={cursor_id} {cursor_kind} [{file_path}]' + cursor_label = f'ID:{cursor_id} {cursor_kind} [{file_path}]' - f.write(indent2(level) + cursor_label + '\n') + f.write(indent(level) + cursor_label + newline()) for child in cursor.get_children(): - cursor_id = _output_cursor_and_children_text( - child, f, level + 1, cursor_id + 1) - return cursor_id + _output_cursor_and_children_text( + child, ast, f, level + 1) -def _output_cursor_and_children_print(cursor: Cursor, level: int = 0, cursor_id: int = 0) -> int: +def _output_cursor_and_children_print(cursor: Cursor, ast: AST, level: int = 0) -> None: """ Prints the current cursor and its child elements in text format, with highlighting for folder, case and node type. @param cursor: The current node of the AST as a libclang cursor object. + @param ast: AST from ast_handler.py @param f: Open file object for output. @param level: The current depth in the AST for indentation purposes. - @param cursor_id: A unique ID to identify each cursor. @return: The current cursor ID for subsequent nodes. """ - cursor_id = Scanner.get_node_id(cursor) - 1 + cursor_id = ast.get_node_id(cursor) cursor_kind = f"" file_path = cursor.location.file.name if cursor.location.file else "" - cursor_label = ( - f'ID={cursor_id} {cursor.spelling} {cursor_kind} {file_path}' - if cursor.spelling else - f'ID={cursor_id} {cursor_kind} {file_path}' - ) - print(indent2(level) + cursor_label) + if cursor.spelling: + cursor_label = (f'ID:{cursor_id} {cursor.spelling} ' + f'{cursor_kind} ' + f'{file_path}') + else: + cursor_label = f'ID:{cursor_id} {cursor_kind} [{file_path}]' + + print(indent(level) + cursor_label) for child in cursor.get_children(): - cursor_id = _output_cursor_and_children_print( - child, level + 1, cursor_id + 1) - return cursor_id + _output_cursor_and_children_print( + child, ast, level + 1) def _output_cursor_and_children_graphviz_digraph(cursor: Cursor, graph: Digraph, max_d: int, current_d: int, parent_node: str = None) -> None: @@ -141,12 +159,15 @@ def _output_cursor_and_children_graphviz_digraph(cursor: Cursor, graph: Digraph, @param cursor: The current node of the AST as a Cursor object from libclang. @param graph: Graphviz Digraph object where the nodes and edges will be added. + @param max_d: Maximal depth. + @param current_d: Current depth. @param parent_node: Name of the parent node in the graph (None for the root node). """ + if current_d > max_d: return - node_label = f"{cursor.kind.name}\n({cursor.spelling})" if cursor.spelling else cursor.kind.name + node_label = f"{cursor.kind.name}{newline()}({cursor.spelling})" if cursor.spelling else cursor.kind.name current_node = f"{cursor.kind.name}_{cursor.hash}" @@ -156,7 +177,7 @@ def _output_cursor_and_children_graphviz_digraph(cursor: Cursor, graph: Digraph, graph.edge(parent_node, current_node) if cursor.kind.is_reference(): - referenced_label = f"ref_to_{cursor.referenced.kind.name}\n({cursor.referenced.spelling})" + referenced_label = f"ref_to_{cursor.referenced.kind.name}{newline()}({cursor.referenced.spelling})" referenced_node = f"ref_{cursor.referenced.hash}" graph.node(referenced_node, label=referenced_label) graph.edge(current_node, referenced_node) diff --git a/pycode/memilio-generation/memilio/generation/scanner.py b/pycode/memilio-generation/memilio/generation/scanner.py index 785fe4f3f1..34b11c6d55 100755 --- a/pycode/memilio-generation/memilio/generation/scanner.py +++ b/pycode/memilio-generation/memilio/generation/scanner.py @@ -38,6 +38,7 @@ from memilio.generation import IntermediateRepresentation, utility + if TYPE_CHECKING: from memilio.generation import ScannerConfig @@ -46,7 +47,6 @@ class Scanner: """ Analyze the model and extract the needed information. """ - cursor_ids = [] def __init__(self: Self, conf: ScannerConfig) -> None: """ @@ -57,105 +57,9 @@ def __init__(self: Self, conf: ScannerConfig) -> None: self.config = conf utility.try_set_libclang_path( self.config.optional.get("libclang_library_path")) - self.ast = None self.node_counter = 0 - self.cursor_nodes = [] - self.create_ast() - - def create_ast(self: Self) -> None: - """ - Create an abstract syntax tree for the main model.cpp file with a corresponding CompilationDatabase. - A compile_commands.json is required (automatically generated in the build process). - """ - idx = Index.create() - - # Create the cmd arguments - file_args = [] - - dirname = utility.try_get_compilation_database_path( - self.config.skbuild_path_to_database) - compdb = CompilationDatabase.fromDirectory(dirname) - commands = compdb.getCompileCommands(self.config.source_file) - for command in commands: - for argument in command.arguments: - if (argument != '-Wno-unknown-warning' and - argument != "--driver-mode=g++" and argument != "-O3" and argument != "-Werror" and argument != "-Wshadow"): - file_args.append(argument) - file_args = file_args[1:-4] - - # Removing only the first and last arguments could miss important flags. - # Safer approach is to include all relevant arguments except for optimization/debug flags - # file_args = file_args[1:-4] if len(file_args) > 5 else file_args - - clang_cmd = [ - "clang-14", self.config.source_file, - "-std=c++17", '-emit-ast', '-o', '-'] - clang_cmd.extend(file_args) - - try: - clang_cmd_result = subprocess.run( - clang_cmd, stdout=subprocess.PIPE) - clang_cmd_result.check_returncode() - except subprocess.CalledProcessError as e: - # Capture standard error and output - logging.error( - f"Clang failed with return code {e.returncode}. Error: {clang_cmd_result.stderr.decode()}") - raise RuntimeError( - f"Clang AST generation failed. See error log for details.") - - # Since `clang.Index.read` expects a file path, write generated abstract syntax tree to a - # temporary named file. This file will be automatically deleted when closed. - with tempfile.NamedTemporaryFile() as ast_file: - ast_file.write(clang_cmd_result.stdout) - self.ast = idx.read(ast_file.name) - - self.assing_ast_with_ids(self.ast.cursor) - - logging.info("AST generation completed successfully.") - - def assing_ast_with_ids(self, cursor: Cursor) -> None: - """ - Traverse the AST and assign a unique ID to each node during traversal. - - Parameters: - @cursor: The current node (Cursor) in the AST to traverse. - """ - - self.node_counter += 1 - cursor_id = self.node_counter - - self.cursor_ids.append((cursor, cursor_id)) - self.cursor_nodes.append(cursor) - - logging.debug( - f"Node {cursor.spelling or cursor.kind} assigned ID {cursor_id}") - - for child in cursor.get_children(): - self.assing_ast_with_ids(child) - - @staticmethod - def get_node_id(cursor: Cursor) -> int: - """ - Returns the id of the current node. - """ - - for c, id in Scanner.cursor_ids: - if c == cursor: - return id - raise IndexError(f"Cursor {cursor} is out of bounds.") - - def get_node_by_index(self, index: int) -> Cursor: - """ - Returns the node at the specified index position. - If the index is out of bounds, None is returned. - """ - if 0 <= index < len(self.cursor_nodes): - return self.cursor_nodes[index] - raise IndexError( - f"Index {index} is out of bounds (0 to {len(self.cursor_nodes) - 1}).") - - def extract_results(self: Self) -> IntermediateRepresentation: + def extract_results(self: Self, root_cursor: Cursor) -> IntermediateRepresentation: """ Extract the information of the abstract syntax tree and save them in the dataclass intermed_repr. Call find_node to visit all nodes of abstract syntax tree and finalize to finish the extraction. @@ -163,9 +67,7 @@ def extract_results(self: Self) -> IntermediateRepresentation: @return Information extracted from the model saved as an IntermediateRepresentation. """ intermed_repr = IntermediateRepresentation() - # graph_visualization._output_cursor_print(self.ast.cursor, 1) - self.find_node(self.ast.cursor, intermed_repr) - # self.output_ast_file() + self.find_node(root_cursor, intermed_repr) self.finalize(intermed_repr) return intermed_repr diff --git a/pycode/memilio-generation/memilio/generation/template/template_cpp.txt b/pycode/memilio-generation/memilio/generation/template/template_cpp.txt index 265ab0e726..a58d9efdb7 100644 --- a/pycode/memilio-generation/memilio/generation/template/template_cpp.txt +++ b/pycode/memilio-generation/memilio/generation/template/template_cpp.txt @@ -51,5 +51,19 @@ PYBIND11_MODULE(_simulation_${python_module_name}, m) ${simulation_graph} + m.def("interpolate_simulation_result", + static_cast (*)(const mio::TimeSeries&, const double)>( + &mio::interpolate_simulation_result), + py::arg("ts"), py::arg("abs_tol") = 1e-14); + + m.def("interpolate_simulation_result", + static_cast (*)(const mio::TimeSeries&, const std::vector&)>( + &mio::interpolate_simulation_result), + py::arg("ts"), py::arg("interpolation_times")); + + m.def("interpolate_ensemble_results", &mio::interpolate_ensemble_results>); + + + m.attr("__version__") = "dev"; } diff --git a/pycode/memilio-generation/memilio/generation/utility.py b/pycode/memilio-generation/memilio/generation/utility.py index 72ace6a808..f5c67fbcfe 100644 --- a/pycode/memilio-generation/memilio/generation/utility.py +++ b/pycode/memilio-generation/memilio/generation/utility.py @@ -114,105 +114,3 @@ def get_base_class_string(base_type: Type) -> List[Any]: result.append(get_base_class_string( base_type.get_template_argument_type(i))) return result - - -def indent(spaces: int) -> str: - """ - Indentation string for pretty-printing. - - @param spaces Number of spaces. - """ - return ' '*spaces - - -def output_cursor_print(cursor: Cursor, spaces: int) -> None: - """ - Low level cursor output to the terminal. - - @param cursor Represents the current node of the AST as an Cursor object from libclang. - @param spaces Number of spaces. - """ - spelling = '' - displayname = '' - - if cursor.spelling: - spelling = cursor.spelling - if cursor.displayname: - displayname = cursor.displayname - kind = cursor.kind - - print(indent(spaces) + spelling, '<' + str(kind) + '>') - print(indent(spaces+1) + '"' + displayname + '"') - - -def output_cursor_and_children_print(cursor: Cursor, spaces: int = 0) -> None: - """ - Output this cursor and its children with minimal formatting to the terminal. - - @param cursor Represents the current node of the AST as an Cursor object from libclang. - @param spaces [Default = 0] Number of spaces. - """ - output_cursor_print(cursor, spaces) - if cursor.kind.is_reference(): - print(indent(spaces) + 'reference to:') - output_cursor_print(cursor.referenced, spaces+1) - - # Recurse for children of this cursor - has_children = False - for c in cursor.get_children(): - if not has_children: - print(indent(spaces) + '{') - has_children = True - output_cursor_and_children_print(c, spaces+1) - - if has_children: - print(indent(spaces) + '}') - - -def output_cursor_file(cursor: Cursor, f: TextIO, spaces: int) -> None: - """ - Low level cursor output to a file. - - @param cursor Represents the current node of the AST as an Cursor object from libclang. - @param f Open file object for output. - @param spaces Number of spaces. - """ - spelling = '' - displayname = '' - - if cursor.spelling: - spelling = cursor.spelling - if cursor.displayname: - displayname = cursor.displayname - kind = cursor.kind - - f.write(indent(spaces) + spelling + ' <' + str(kind) + '> ') - if cursor.location.file: - f.write(cursor.location.file.name + '\n') - f.write(indent(spaces+1) + '"' + displayname + '"\n') - - -def output_cursor_and_children_file( - cursor: Cursor, f: TextIO, spaces: int = 0) -> None: - """ - Output this cursor and its children with minimal formatting to a file. - - @param cursor Represents the current node of the AST as an Cursor object from libclang. - @param f Open file object for output. - @param spaces Number of spaces. - """ - output_cursor_file(cursor, f, spaces) - if cursor.kind.is_reference(): - f.write(indent(spaces) + 'reference to:\n') - output_cursor_file(cursor.referenced, f, spaces+1) - - # Recurse for children of this cursor - has_children = False - for c in cursor.get_children(): - if not has_children: - f.write(indent(spaces) + '{\n') - has_children = True - output_cursor_and_children_file(c, f, spaces+1) - - if has_children: - f.write(indent(spaces) + '}\n') From b2669b54d8307f31c62bae9031a484df69c0f8d1 Mon Sep 17 00:00:00 2001 From: Daniel Richter Date: Mon, 4 Nov 2024 10:36:57 +0100 Subject: [PATCH 07/12] Changes in ast.py and graph_visualization.py. --- .../generation/{ast_handler.py => ast.py} | 9 +++++++- .../memilio/generation/graph_visualization.py | 22 +++++++++++-------- 2 files changed, 21 insertions(+), 10 deletions(-) rename pycode/memilio-generation/memilio/generation/{ast_handler.py => ast.py} (96%) diff --git a/pycode/memilio-generation/memilio/generation/ast_handler.py b/pycode/memilio-generation/memilio/generation/ast.py similarity index 96% rename from pycode/memilio-generation/memilio/generation/ast_handler.py rename to pycode/memilio-generation/memilio/generation/ast.py index 1d3d4b5617..82b6778e78 100644 --- a/pycode/memilio-generation/memilio/generation/ast_handler.py +++ b/pycode/memilio-generation/memilio/generation/ast.py @@ -17,7 +17,10 @@ # See the License for the specific language governing permissions and # limitations under the License. ############################################################################# - +""" +@file ast.py +@brief Create the ast and assign ids. Get ids and nodes. +""" import os import subprocess import tempfile @@ -34,6 +37,10 @@ class AST: + """ + Create the ast and assign ids. + Functions for getting nodes and node ids. + """ def __init__(self: Self, conf: "ScannerConfig") -> None: self.config = conf diff --git a/pycode/memilio-generation/memilio/generation/graph_visualization.py b/pycode/memilio-generation/memilio/generation/graph_visualization.py index 19a7e98fc9..7d63a20a86 100644 --- a/pycode/memilio-generation/memilio/generation/graph_visualization.py +++ b/pycode/memilio-generation/memilio/generation/graph_visualization.py @@ -17,12 +17,15 @@ # See the License for the specific language governing permissions and # limitations under the License. ############################################################################# - +""" +@file graph_visualization.py +@brief Class for plotting the abstract-syntax-tree in different formats. +""" import os from typing import TextIO from graphviz import Digraph from clang.cindex import Cursor -from .ast_handler import AST +from .ast import AST class Visualization: @@ -94,10 +97,11 @@ def newline() -> str: def _output_cursor_and_children_text(cursor: Cursor, ast: AST, f: TextIO, level: int = 0) -> int: """ - Output of the cursor and its children in text format, with highlighting for folder, spelling and child type. + Output of the cursor and its children in text format, + with highlighting for folder, spelling and child type. @param cursor: The current node of the AST as a cursor object from libclang. - @param ast: AST from ast_handler.py + @param ast: ast object from AST class. @param f: Open file object for output. @param level: The current depth in the AST for indentation purposes. @param cursor_id: A unique ID to identify each cursor. @@ -113,7 +117,7 @@ def _output_cursor_and_children_text(cursor: Cursor, ast: AST, f: TextIO, level: f'{cursor_kind} ' f'{file_path}') else: - cursor_label = f'ID:{cursor_id} {cursor_kind} [{file_path}]' + cursor_label = f'ID:{cursor_id} {cursor_kind} {file_path}' f.write(indent(level) + cursor_label + newline()) @@ -124,11 +128,11 @@ def _output_cursor_and_children_text(cursor: Cursor, ast: AST, f: TextIO, level: def _output_cursor_and_children_print(cursor: Cursor, ast: AST, level: int = 0) -> None: """ - Prints the current cursor and its child elements in text format, - with highlighting for folder, case and node type. + Prints the current cursor and its child elements in text format into the terminal, + with highlighting for folder, spelling and child type. @param cursor: The current node of the AST as a libclang cursor object. - @param ast: AST from ast_handler.py + @param ast: ast object from AST class. @param f: Open file object for output. @param level: The current depth in the AST for indentation purposes. @return: The current cursor ID for subsequent nodes. @@ -144,7 +148,7 @@ def _output_cursor_and_children_print(cursor: Cursor, ast: AST, level: int = 0) f'{cursor_kind} ' f'{file_path}') else: - cursor_label = f'ID:{cursor_id} {cursor_kind} [{file_path}]' + cursor_label = f'ID:{cursor_id} {cursor_kind} {file_path}' print(indent(level) + cursor_label) From 57872640ed8f62726417986d2984c6513ce9ce9f Mon Sep 17 00:00:00 2001 From: Daniel Richter Date: Mon, 4 Nov 2024 11:04:34 +0100 Subject: [PATCH 08/12] Changes to ast and test_oseir_generation --- pycode/memilio-generation/memilio/generation/ast.py | 1 - .../memilio/generation_test/test_oseir_generation.py | 7 ++++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pycode/memilio-generation/memilio/generation/ast.py b/pycode/memilio-generation/memilio/generation/ast.py index 82b6778e78..bb29954e92 100644 --- a/pycode/memilio-generation/memilio/generation/ast.py +++ b/pycode/memilio-generation/memilio/generation/ast.py @@ -21,7 +21,6 @@ @file ast.py @brief Create the ast and assign ids. Get ids and nodes. """ -import os import subprocess import tempfile import logging diff --git a/pycode/memilio-generation/memilio/generation_test/test_oseir_generation.py b/pycode/memilio-generation/memilio/generation_test/test_oseir_generation.py index 5913acf8de..69c6d326bf 100644 --- a/pycode/memilio-generation/memilio/generation_test/test_oseir_generation.py +++ b/pycode/memilio-generation/memilio/generation_test/test_oseir_generation.py @@ -24,7 +24,7 @@ import unittest from unittest.mock import patch -from memilio.generation import Generator, Scanner, ScannerConfig +from memilio.generation import Generator, Scanner, ScannerConfig, ast class TestOseirGeneration(unittest.TestCase): @@ -71,9 +71,10 @@ def setUp(self, try_get_compilation_database_path_mock): conf = ScannerConfig.from_dict(config_json) self.scanner = Scanner(conf) + self.ast = ast.AST(conf) def test_clean_oseir(self): - irdata = self.scanner.extract_results() + irdata = self.scanner.extract_results(self.ast.root_cursor) generator = Generator() generator.create_substitutions(irdata) @@ -87,7 +88,7 @@ def test_clean_oseir(self): def test_wrong_model_name(self): self.scanner.config.model_class = "wrong_name" with self.assertRaises(AssertionError) as error: - self.scanner.extract_results() + self.scanner.extract_results(self.ast.root_cursor) error_message = "set a model name" self.assertEqual(str(error.exception), error_message) From 8ff0403dd51f6d8357c5c2dd2863bb2b80ecbb1c Mon Sep 17 00:00:00 2001 From: Daniel Richter Date: Mon, 4 Nov 2024 12:38:42 +0100 Subject: [PATCH 09/12] template changes --- .../memilio/generation/template/template_cpp.txt | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/pycode/memilio-generation/memilio/generation/template/template_cpp.txt b/pycode/memilio-generation/memilio/generation/template/template_cpp.txt index a58d9efdb7..5cf8c6f74c 100644 --- a/pycode/memilio-generation/memilio/generation/template/template_cpp.txt +++ b/pycode/memilio-generation/memilio/generation/template/template_cpp.txt @@ -51,15 +51,11 @@ PYBIND11_MODULE(_simulation_${python_module_name}, m) ${simulation_graph} - m.def("interpolate_simulation_result", - static_cast (*)(const mio::TimeSeries&, const double)>( - &mio::interpolate_simulation_result), - py::arg("ts"), py::arg("abs_tol") = 1e-14); + m.def("interpolate_simulation_result",static_cast (*)(const mio::TimeSeries&, const double)>(&mio::interpolate_simulation_result), + py::arg("ts"), py::arg("abs_tol") = 1e-14); - m.def("interpolate_simulation_result", - static_cast (*)(const mio::TimeSeries&, const std::vector&)>( - &mio::interpolate_simulation_result), - py::arg("ts"), py::arg("interpolation_times")); + m.def("interpolate_simulation_result",static_cast (*)(const mio::TimeSeries&, const std::vector&)>(&mio::interpolate_simulation_result), + py::arg("ts"), py::arg("interpolation_times")); m.def("interpolate_ensemble_results", &mio::interpolate_ensemble_results>); From eedfec83d94bd54f4a2a987c8adc6067113d45a0 Mon Sep 17 00:00:00 2001 From: Daniel Richter Date: Mon, 4 Nov 2024 12:44:43 +0100 Subject: [PATCH 10/12] template changes --- .../memilio/generation/template/template_cpp.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pycode/memilio-generation/memilio/generation/template/template_cpp.txt b/pycode/memilio-generation/memilio/generation/template/template_cpp.txt index 5cf8c6f74c..9c3b4b288f 100644 --- a/pycode/memilio-generation/memilio/generation/template/template_cpp.txt +++ b/pycode/memilio-generation/memilio/generation/template/template_cpp.txt @@ -51,13 +51,7 @@ PYBIND11_MODULE(_simulation_${python_module_name}, m) ${simulation_graph} - m.def("interpolate_simulation_result",static_cast (*)(const mio::TimeSeries&, const double)>(&mio::interpolate_simulation_result), - py::arg("ts"), py::arg("abs_tol") = 1e-14); - m.def("interpolate_simulation_result",static_cast (*)(const mio::TimeSeries&, const std::vector&)>(&mio::interpolate_simulation_result), - py::arg("ts"), py::arg("interpolation_times")); - - m.def("interpolate_ensemble_results", &mio::interpolate_ensemble_results>); From 746a86b8feac007acac39adff7146ba65cc39150 Mon Sep 17 00:00:00 2001 From: Daniel Richter Date: Mon, 4 Nov 2024 12:55:09 +0100 Subject: [PATCH 11/12] init, test and template changes --- pycode/memilio-generation/memilio/generation/__init__.py | 1 + .../memilio/generation/template/template_cpp.txt | 4 ---- .../memilio/generation_test/test_oseir_generation.py | 4 ++-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/pycode/memilio-generation/memilio/generation/__init__.py b/pycode/memilio-generation/memilio/generation/__init__.py index 65098b9693..3289ebf7be 100644 --- a/pycode/memilio-generation/memilio/generation/__init__.py +++ b/pycode/memilio-generation/memilio/generation/__init__.py @@ -25,3 +25,4 @@ from .intermediate_representation import IntermediateRepresentation from .scanner import Scanner from .scanner_config import ScannerConfig +from .ast import AST diff --git a/pycode/memilio-generation/memilio/generation/template/template_cpp.txt b/pycode/memilio-generation/memilio/generation/template/template_cpp.txt index 9c3b4b288f..265ab0e726 100644 --- a/pycode/memilio-generation/memilio/generation/template/template_cpp.txt +++ b/pycode/memilio-generation/memilio/generation/template/template_cpp.txt @@ -51,9 +51,5 @@ PYBIND11_MODULE(_simulation_${python_module_name}, m) ${simulation_graph} - - - - m.attr("__version__") = "dev"; } diff --git a/pycode/memilio-generation/memilio/generation_test/test_oseir_generation.py b/pycode/memilio-generation/memilio/generation_test/test_oseir_generation.py index 69c6d326bf..c08a218184 100644 --- a/pycode/memilio-generation/memilio/generation_test/test_oseir_generation.py +++ b/pycode/memilio-generation/memilio/generation_test/test_oseir_generation.py @@ -24,7 +24,7 @@ import unittest from unittest.mock import patch -from memilio.generation import Generator, Scanner, ScannerConfig, ast +from memilio.generation import Generator, Scanner, ScannerConfig, AST class TestOseirGeneration(unittest.TestCase): @@ -71,7 +71,7 @@ def setUp(self, try_get_compilation_database_path_mock): conf = ScannerConfig.from_dict(config_json) self.scanner = Scanner(conf) - self.ast = ast.AST(conf) + self.ast = AST(conf) def test_clean_oseir(self): irdata = self.scanner.extract_results(self.ast.root_cursor) From 6f8a5356820d6f700c1b34241f6c41367161f65b Mon Sep 17 00:00:00 2001 From: Daniel Richter Date: Wed, 6 Nov 2024 14:14:37 +0100 Subject: [PATCH 12/12] Changes ast.py , graph_viz.py , scanner.py , example_oseir.py Review changes. --- .../memilio/generation/ast.py | 40 +++--- .../memilio/generation/graph_visualization.py | 121 +++++++----------- .../memilio/generation/scanner.py | 51 +++----- .../memilio/tools/example_oseir.py | 9 +- 4 files changed, 87 insertions(+), 134 deletions(-) diff --git a/pycode/memilio-generation/memilio/generation/ast.py b/pycode/memilio-generation/memilio/generation/ast.py index bb29954e92..77a0e8d837 100644 --- a/pycode/memilio-generation/memilio/generation/ast.py +++ b/pycode/memilio-generation/memilio/generation/ast.py @@ -1,7 +1,7 @@ ############################################################################# # Copyright (C) 2020-2024 MEmilio # -# Authors: Daniel Richter +# Authors: Maximilian Betz, Daniel Richter # # Contact: Martin J. Kuehn # @@ -36,8 +36,7 @@ class AST: - """ - Create the ast and assign ids. + """! Create the ast and assign ids. Functions for getting nodes and node ids. """ @@ -50,30 +49,31 @@ def __init__(self: Self, conf: "ScannerConfig") -> None: self.translation_unit = self.create_ast() def create_ast(self: Self) -> TranslationUnit: - """ - Create an abstract syntax tree for the main model.cpp file with a corresponding CompilationDatabase. + """! Create an abstract syntax tree for the main model.cpp file with a corresponding CompilationDatabase. A compile_commands.json is required (automatically generated in the build process). """ + self.cursor_id = -1 + self.id_to_val.clear() + self.val_to_id.clear() + idx = Index.create() - # Create the cmd arguments file_args = [] + unwanted_arguments = [ + '-Wno-unknown-warning', "--driver-mode=g++", "-O3", "-Werror", "-Wshadow" + ] + dirname = utility.try_get_compilation_database_path( self.config.skbuild_path_to_database) compdb = CompilationDatabase.fromDirectory(dirname) commands = compdb.getCompileCommands(self.config.source_file) for command in commands: for argument in command.arguments: - if (argument != '-Wno-unknown-warning' and - argument != "--driver-mode=g++" and argument != "-O3" and argument != "-Werror" and argument != "-Wshadow"): + if argument not in unwanted_arguments: file_args.append(argument) file_args = file_args[1:-4] - # Removing only the first and last arguments could miss important flags. - # Safer approach is to include all relevant arguments except for optimization/debug flags - # file_args = file_args[1:-4] if len(file_args) > 5 else file_args - clang_cmd = [ "clang-14", self.config.source_file, "-std=c++17", '-emit-ast', '-o', '-'] @@ -96,15 +96,14 @@ def create_ast(self: Self) -> TranslationUnit: ast_file.write(clang_cmd_result.stdout) translation_unit = idx.read(ast_file.name) - self.assing_ast_with_ids(translation_unit.cursor) + self._assing_ast_with_ids(translation_unit.cursor) logging.info("AST generation completed successfully.") return translation_unit - def assing_ast_with_ids(self, cursor: Cursor) -> None: - """ - Traverse the AST and assign a unique ID to each node during traversal. + def _assing_ast_with_ids(self, cursor: Cursor) -> None: + """! Traverse the AST and assign a unique ID to each node during traversal. @param cursor: The current node (Cursor) in the AST to traverse. """ @@ -122,15 +121,15 @@ def assing_ast_with_ids(self, cursor: Cursor) -> None: f"Node {cursor.spelling or cursor.kind} assigned ID {id}") for child in cursor.get_children(): - self.assing_ast_with_ids(child) + self._assing_ast_with_ids(child) @property def root_cursor(self): return self.translation_unit.cursor def get_node_id(self, cursor: Cursor) -> int: - """ - Returns the id of the current node. + """! Returns the id of the current node. + Extracs the key from the current cursor from the dictonary id_to_val @param cursor: The current node of the AST as a cursor object from libclang. @@ -143,8 +142,7 @@ def get_node_id(self, cursor: Cursor) -> int: raise IndexError(f"Cursor {cursor} is out of bounds.") def get_node_by_index(self, index: int) -> Cursor: - """ - Returns the node at the specified index position. + """! Returns the node at the specified index position. @param index: Node_id from the ast. """ diff --git a/pycode/memilio-generation/memilio/generation/graph_visualization.py b/pycode/memilio-generation/memilio/generation/graph_visualization.py index 7d63a20a86..9ef18c8964 100644 --- a/pycode/memilio-generation/memilio/generation/graph_visualization.py +++ b/pycode/memilio-generation/memilio/generation/graph_visualization.py @@ -1,7 +1,7 @@ ############################################################################# # Copyright (C) 2020-2024 MEmilio # -# Authors: Daniel Richter +# Authors: Maximilian Betz, Daniel Richter # # Contact: Martin J. Kuehn # @@ -17,38 +17,48 @@ # See the License for the specific language governing permissions and # limitations under the License. ############################################################################# -""" -@file graph_visualization.py -@brief Class for plotting the abstract-syntax-tree in different formats. -""" + import os -from typing import TextIO +import logging +from typing import Callable from graphviz import Digraph from clang.cindex import Cursor -from .ast import AST +from memilio.generation.ast import AST class Visualization: - """ - Class for plotting the abstract-syntax-tree in different formats. + """! Class for plotting the abstract syntax tree in different formats. """ @staticmethod def output_ast_terminal(ast: AST, cursor: Cursor) -> None: - """ - Output the abstract syntax tree to terminal. + """! Output the abstract syntax tree to terminal. @param ast: ast object from AST class. @param cursor: The current node of the AST as a cursor object from libclang. """ - _output_cursor_and_children_print(cursor, ast) + def terminal_writer(level: int, cursor_label: str) -> None: + print(indent(level) + cursor_label) - print(f'AST-terminal written.') + _output_cursor_and_children(cursor, ast, terminal_writer) + + logging.info("AST-Terminal written.") @staticmethod - def output_ast_png(cursor: Cursor, max_depth: int, ) -> None: - """ - Output the abstract syntax tree to a .png. Set the starting node and the max depth. + def output_ast_png(cursor: Cursor, max_depth: int, output_file_name: str = 'ast_graph') -> None: + """! Output the abstract syntax tree to a .png. Set the starting node and the max depth. + + To save the abstract syntax tree as an png with a starting node and a depth u cann use the following command + + Example command: aviz.output_ast_png(ast.get_node_by_index(1), 2) + + aviz -> instance of the Visualization class. + + ast -> instance of the AST class. + + .get_node_by_index -> get a specific node by id (use .output_ast_formatted to see node ids) + + The number 2 is a example for the depth the graph will show @param cursor: The current node of the AST as a cursor object from libclang. @param max_depth: Maximal depth the graph displays. @@ -59,83 +69,47 @@ def output_ast_png(cursor: Cursor, max_depth: int, ) -> None: _output_cursor_and_children_graphviz_digraph( cursor, graph, max_depth, 0) - output_file = 'ast_graph' - - graph.render(output_file, view=False) + graph.render(filename=output_file_name, view=False) - output_path = os.path.abspath(f"{output_file}.png") - print(f'AST-png written to {output_path}') + output_path = os.path.abspath(f"{output_file_name}.png") + logging.info(f"AST-png written to {output_path}") @staticmethod - def output_ast_formatted(ast: AST, cursor: Cursor) -> None: - """ - Output the abstract syntax tree to a file. + def output_ast_formatted(ast: AST, cursor: Cursor, output_file_name: str = 'ast_formated.txt') -> None: + """!Output the abstract syntax tree to a file. @param ast: ast object from AST class. @param cursor: The current node of the AST as a cursor object from libclang. """ - with open('ast_formated.txt', 'w') as f: - _output_cursor_and_children_text(cursor, ast, f) + with open(output_file_name, 'w') as f: + def file_writer(level: int, cursor_label: str) -> None: + f.write(indent(level) + cursor_label + newline()) + _output_cursor_and_children(cursor, ast, file_writer) - print("AST-formated written to " + str(os.path.abspath(f.name))) + output_path = os.path.abspath(f"{output_file_name}") + logging.info(f"AST-formated written to {output_path}") def indent(level: int) -> str: - """ - Create an indentation based on the level. + """! Create an indentation based on the level. """ return '│ ' * level + '├── ' def newline() -> str: - """ - Create a new line. + """! Create a new line. """ return '\n' -def _output_cursor_and_children_text(cursor: Cursor, ast: AST, f: TextIO, level: int = 0) -> int: - """ - Output of the cursor and its children in text format, - with highlighting for folder, spelling and child type. - - @param cursor: The current node of the AST as a cursor object from libclang. - @param ast: ast object from AST class. - @param f: Open file object for output. - @param level: The current depth in the AST for indentation purposes. - @param cursor_id: A unique ID to identify each cursor. - """ - - cursor_id = ast.get_node_id(cursor) - - cursor_kind = f"" - file_path = cursor.location.file.name if cursor.location.file else "" - - if cursor.spelling: - cursor_label = (f'ID:{cursor_id} {cursor.spelling} ' - f'{cursor_kind} ' - f'{file_path}') - else: - cursor_label = f'ID:{cursor_id} {cursor_kind} {file_path}' - - f.write(indent(level) + cursor_label + newline()) - - for child in cursor.get_children(): - _output_cursor_and_children_text( - child, ast, f, level + 1) - - -def _output_cursor_and_children_print(cursor: Cursor, ast: AST, level: int = 0) -> None: - """ - Prints the current cursor and its child elements in text format into the terminal, - with highlighting for folder, spelling and child type. +def _output_cursor_and_children(cursor: Cursor, ast: AST, writer: Callable[[int, str], None], level: int = 0) -> None: + """!Generic function to output the cursor and its children with a specified writer. @param cursor: The current node of the AST as a libclang cursor object. - @param ast: ast object from AST class. - @param f: Open file object for output. + @param ast: AST object from the AST class. + @param writer: Function that takes `level` and `cursor_label` and handles output. @param level: The current depth in the AST for indentation purposes. - @return: The current cursor ID for subsequent nodes. """ cursor_id = ast.get_node_id(cursor) @@ -150,16 +124,15 @@ def _output_cursor_and_children_print(cursor: Cursor, ast: AST, level: int = 0) else: cursor_label = f'ID:{cursor_id} {cursor_kind} {file_path}' - print(indent(level) + cursor_label) + writer(level, cursor_label) for child in cursor.get_children(): - _output_cursor_and_children_print( - child, ast, level + 1) + _output_cursor_and_children( + child, ast, writer, level + 1) def _output_cursor_and_children_graphviz_digraph(cursor: Cursor, graph: Digraph, max_d: int, current_d: int, parent_node: str = None) -> None: - """ - Output the cursor and its children as a graph using Graphviz. + """! Output the cursor and its children as a graph using Graphviz. @param cursor: The current node of the AST as a Cursor object from libclang. @param graph: Graphviz Digraph object where the nodes and edges will be added. diff --git a/pycode/memilio-generation/memilio/generation/scanner.py b/pycode/memilio-generation/memilio/generation/scanner.py index 34b11c6d55..317ce59889 100755 --- a/pycode/memilio-generation/memilio/generation/scanner.py +++ b/pycode/memilio-generation/memilio/generation/scanner.py @@ -24,14 +24,7 @@ from __future__ import annotations import os -import subprocess -import sys -import tempfile -import logging -from graphviz import Digraph from typing import TYPE_CHECKING, Any, Callable -from warnings import catch_warnings - from clang.cindex import * from typing_extensions import Self @@ -44,8 +37,7 @@ class Scanner: - """ - Analyze the model and extract the needed information. + """! Analyze the model and extract the needed information. """ def __init__(self: Self, conf: ScannerConfig) -> None: @@ -57,13 +49,12 @@ def __init__(self: Self, conf: ScannerConfig) -> None: self.config = conf utility.try_set_libclang_path( self.config.optional.get("libclang_library_path")) - self.node_counter = 0 def extract_results(self: Self, root_cursor: Cursor) -> IntermediateRepresentation: - """ - Extract the information of the abstract syntax tree and save them in the dataclass intermed_repr. + """! Extract the information of the abstract syntax tree and save them in the dataclass intermed_repr. Call find_node to visit all nodes of abstract syntax tree and finalize to finish the extraction. + @param root_cursor Represents the root node of the abstract syntax tree as a Cursor object from libclang. @return Information extracted from the model saved as an IntermediateRepresentation. """ intermed_repr = IntermediateRepresentation() @@ -73,8 +64,7 @@ def extract_results(self: Self, root_cursor: Cursor) -> IntermediateRepresentati def find_node(self: Self, node: Cursor, intermed_repr: IntermediateRepresentation, namespace: str = "") -> None: - """ - Recursively walk over every node of an abstract syntax tree. Save the namespace the node is in. + """! Recursively walk over every node of an abstract syntax tree. Save the namespace the node is in. Call check_node_kind for extracting information from the nodes. @param node Represents the current node of the abstract syntax tree as a Cursor object from libclang. @@ -92,8 +82,7 @@ def find_node(self: Self, node: Cursor, def switch_node_kind(self: Self, kind: CursorKind) -> Callable[[Any, IntermediateRepresentation], None]: - """ - Dictionary to map CursorKind to methods. Works like a switch. + """! Dictionary to map CursorKind to methods. Works like a switch. @param Underlying kind of the current node. @return Appropriate method for the given kind. @@ -114,8 +103,7 @@ def switch_node_kind(self: Self, kind: CursorKind) -> Callable[[Any, def check_enum( self: Self, node: Cursor, intermed_repr: IntermediateRepresentation) -> None: - """ - Inspect the nodes of kind ENUM_DECL and write needed information into intermed_repr. + """! Inspect the nodes of kind ENUM_DECL and write needed information into intermed_repr. Information: Name of Enum @param node Current node represented as a Cursor object. @@ -127,8 +115,7 @@ def check_enum( def check_enum_const( self: Self, node: Cursor, intermed_repr: IntermediateRepresentation) -> None: - """ - Inspect the nodes of kind ENUM_CONSTANT_DECL and write needed information into intermed_repr. + """! Inspect the nodes of kind ENUM_CONSTANT_DECL and write needed information into intermed_repr. Information: Keys of an Enum @param node Current node represented as a Cursor object. @@ -141,8 +128,7 @@ def check_enum_const( def check_class( self: Self, node: Cursor, intermed_repr: IntermediateRepresentation) -> None: - """ - Inspect the nodes of kind CLASS_DECL and write information + """! Inspect the nodes of kind CLASS_DECL and write information (model_class, model_base, simulation_class, parameterset_wrapper) into intermed_repr. @param node Current node represented as a Cursor object. @@ -163,8 +149,7 @@ def check_class( def check_model_base( self: Self, node: Cursor, intermed_repr: IntermediateRepresentation) -> None: - """ - Helper function to retreive the model base. + """! Helper function to retreive the model base. @param node Current node represented as a Cursor object. @param intermed_repr Dataclass used for saving the extracted model features. @@ -178,8 +163,7 @@ def check_model_base( def check_base_specifier( self: Self, node: Cursor, intermed_repr: IntermediateRepresentation) -> None: - """ - Not used yet. + """! Not used yet. Inspect nodes which represent base specifier. For now this is handled by the parent node, which represents the class. """ @@ -188,8 +172,7 @@ def check_base_specifier( def check_model_includes( self: Self, node: Cursor, intermed_repr: IntermediateRepresentation) -> None: - """ - Helper function to retrieve the model specific includes needed for pybind. + """! Helper function to retrieve the model specific includes needed for pybind. @param node Current node represented as a Cursor object. @param intermed_repr Dataclass used for saving the extracted model features. @@ -220,8 +203,7 @@ def check_model_includes( def check_age_group( self: Self, node: Cursor, intermed_repr: IntermediateRepresentation) -> None: - """ - Inspect the nodes of kind CLASS_DECL with the name defined in + """! Inspect the nodes of kind CLASS_DECL with the name defined in config.age_group and write needed information into intermed_repr. Information: age_group @@ -246,8 +228,7 @@ def check_age_group( def check_constructor( self: Self, node: Cursor, intermed_repr: IntermediateRepresentation) -> None: - """ - Inspect the nodes of kind CONSTRUCTOR and write needed information into intermed_repr. + """! Inspect the nodes of kind CONSTRUCTOR and write needed information into intermed_repr. Information: intermed_repr.init @param node Current node represented as a Cursor object. @@ -266,8 +247,7 @@ def check_constructor( def check_type_alias( self: Self, node: Cursor, intermed_repr: IntermediateRepresentation) -> None: - """ - Inspect the nodes of kind TYPE_ALIAS_DECL and write needed information into intermed_repr. + """! Inspect the nodes of kind TYPE_ALIAS_DECL and write needed information into intermed_repr. Information: intermed_repr.parameterset @param node Current node represented as a Cursor object. @@ -283,8 +263,7 @@ def check_struct( pass def finalize(self: Self, intermed_repr: IntermediateRepresentation) -> None: - """ - Finalize the IntermediateRepresenation as last step of the Scanner. + """! Finalize the IntermediateRepresenation as last step of the Scanner. Write needed information from config into intermed_repr, delet unnecesary enums and check for missing model features. diff --git a/pycode/memilio-generation/memilio/tools/example_oseir.py b/pycode/memilio-generation/memilio/tools/example_oseir.py index a3226c5bd9..a2457d9825 100755 --- a/pycode/memilio-generation/memilio/tools/example_oseir.py +++ b/pycode/memilio-generation/memilio/tools/example_oseir.py @@ -31,7 +31,8 @@ # For older python versions import importlib_resources -from memilio.generation import Generator, Scanner, ScannerConfig +from memilio.generation import Generator, Scanner, ScannerConfig, AST +from memilio.generation.graph_visualization import Visualization def run_memilio_generation(print_ast=False): @@ -41,9 +42,11 @@ def run_memilio_generation(print_ast=False): with open(path) as file: conf = ScannerConfig.schema().loads(file.read(), many=True)[0] scanner = Scanner(conf) + ast = AST(conf) + aviz = Visualization() # Extract results of Scanner into a intermed_repr - intermed_repr = scanner.extract_results() + intermed_repr = scanner.extract_results(ast.root_cursor) # Generate code generator = Generator() @@ -52,7 +55,7 @@ def run_memilio_generation(print_ast=False): # Print the abstract syntax tree to a file if (print_ast): - scanner.output_ast_file() + aviz.output_ast_formatted(ast, ast.get_node_by_index(0)) if __name__ == "__main__":