Issue #5658

assemblies/plugins/pom.xml

@@ -943,6 +943,12 @@
             <version>${project.version}</version>
             <type>zip</type>
         </dependency>
+        <dependency>
+            <groupId>org.apache.hop</groupId>
+            <artifactId>hop-transform-dbimpact</artifactId>
+            <version>${project.version}</version>
+            <type>zip</type>
+        </dependency>
         <dependency>
             <groupId>org.apache.hop</groupId>
             <artifactId>hop-transform-dbproc</artifactId>

docs/hop-user-manual/modules/ROOT/nav.adoc

@@ -116,6 +116,7 @@ under the License.
 *** xref:pipeline/transforms/databaselookup.adoc[Database Lookup]
 *** xref:pipeline/transforms/datagrid.adoc[Data Grid]
 *** xref:pipeline/transforms/datasetinput.adoc[Data Set Input]
+*** xref:pipeline/transforms/dbimpactinput.adoc[Database Impact Input]
 *** xref:pipeline/transforms/validator.adoc[Data Validator]
 *** xref:pipeline/transforms/delay.adoc[Delay row]
 *** xref:pipeline/transforms/delete.adoc[Delete]

docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms.adoc

@@ -75,6 +75,8 @@ The pages nested under this topic contain information on how to use the transfor
 * xref:pipeline/transforms/databasejoin.adoc[Database Join]
 * xref:pipeline/transforms/databaselookup.adoc[Database Lookup]
 * xref:pipeline/transforms/datagrid.adoc[Data Grid]
+* xref:pipeline/transforms/datasetinput.adoc[Dataset Input]
+* xref:pipeline/transforms/dbimpactinput.adoc[Database Impact Input]
 * xref:pipeline/transforms/delay.adoc[Delay row]
 * xref:pipeline/transforms/delete.adoc[Delete]
 * xref:pipeline/transforms/serialize-de-from-file.adoc[De-Serialize From File]

docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/dbimpactinput.adoc

@@ -0,0 +1,77 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+  http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:documentationPath: /pipeline/transforms/
+:language: en_US
+:description: Read database impact information from pipelines.
+
+= image:images/icons/database.svg[Database Impact Input, role="image-doc-icon"] Database Impact Input
+
+[%noheader,cols="3a,1a", role="table-no-borders" ]
+|===
+|
+== Description
+
+The Database Impact Input transform allows gives you the impact of a pipeline on a database.
+It explains which transforms read, write, update or delete information, in which table, the SQL used, and so on.
+
+IMPORTANT: This transform supports error handling.  You can add an error handling hop to a transform to be informed of the pipelines that throw an error when calculating the database impact. See also: xref:pipeline/errorhandling.adoc[Pipeline error handling].
+
+|
+== Supported Engines
+[%noheader,cols="2,1a",frame=none, role="table-supported-engines"]
+!===
+!Hop Engine! image:check_mark.svg[Supported, 24]
+!Spark! image:check_mark.svg[Supported, 24]
+!Flink! image:check_mark.svg[Supported, 24]
+!Dataflow! image:check_mark.svg[Supported, 24]
+!===
+|===
+
+== Options
+
+[options="header]
+|===
+|Option|Description
+|Transform name
+|Name of the transform.
+This name has to be unique in a single pipeline.
+
+|File name field
+|The name of the field in the transform input which will contain the filenames to analyze the database impact for.
+
+|===
+
+TIP: You can use a xref:pipeline/transforms/getfilenames.adoc[Get File Names] transform as input.  To search for your project pipelines specify folder `${PROJECT_HOME}` with wildcard `.*\.hpl$`.
+
+== Output fields
+
+[options="header]
+|===
+|Fieldname|Type
+|Type|String
+|PipelineName|String
+|PipelineFileName|String
+|TransformName|String
+|DatabaseName|String
+|DatabaseTable|String
+|TableField|String
+|FieldName|String
+|FieldOrigin|String
+|SQL|String
+|Remark|String
+
+|===

engine/src/main/java/org/apache/hop/pipeline/PipelineMeta.java

@@ -1087,7 +1087,7 @@ public IRowMeta getTransformFields(IVariables variables, TransformMeta[] transfo
 
     for (TransformMeta meta : transformMeta) {
       IRowMeta flds = getTransformFields(variables, meta);
-      if (flds != null) {
+      if (flds != null && meta != null) {
         fields.mergeRowMeta(flds, meta.getName());
       }
     }
@@ -1387,10 +1387,15 @@ public IRowMeta getThisTransformFields(
     IRowMeta[] infoRowMeta;
     TransformMeta[] lu = getInfoTransform(transformMeta);
     if (Utils.isEmpty(lu)) {
-      infoRowMeta =
-          new IRowMeta[] {
-            iTransformMeta.getTableFields(variables),
-          };
+      try {
+        infoRowMeta =
+            new IRowMeta[] {
+              iTransformMeta.getTableFields(variables),
+            };
+      } catch (HopDatabaseException dbe) {
+        throw new HopTransformException(
+            "Error getting table fields in transform " + transformMeta.getName(), dbe);
+      }
     } else {
       infoRowMeta = new IRowMeta[lu.length];
       for (int i = 0; i < lu.length; i++) {
@@ -2632,7 +2637,12 @@ public void analyseImpact(
       if (lu != null) {
         infoRowMeta = getTransformFields(variables, lu);
       } else {
-        infoRowMeta = iTransformMeta.getTableFields(variables);
+        try {
+          infoRowMeta = iTransformMeta.getTableFields(variables);
+        } catch (HopDatabaseException dbe) {
+          throw new HopTransformException(
+              "Error getting table fields from in transform " + transformMeta.getName(), dbe);
+        }
       }
 
       iTransformMeta.analyseImpact(

engine/src/main/java/org/apache/hop/pipeline/transform/BaseTransformMeta.java

@@ -242,7 +242,7 @@ public boolean hasChanged() {
    * @return the table fields
    * @param variables
    */
-  public IRowMeta getTableFields(IVariables variables) {
+  public IRowMeta getTableFields(IVariables variables) throws HopDatabaseException {
     return null;
   }
 

engine/src/main/java/org/apache/hop/pipeline/transform/ITransformMeta.java

@@ -251,7 +251,7 @@ void check(
    * @return The fields used by this transform, this is being used for the Impact analyses.
    * @param variables
    */
-  IRowMeta getTableFields(IVariables variables);
+  IRowMeta getTableFields(IVariables variables) throws HopDatabaseException;
 
   /** This method is added to exclude certain transforms from layout checking. */
   boolean excludeFromRowLayoutVerification();

plugins/transforms/databaselookup/src/main/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookupMeta.java

@@ -361,7 +361,7 @@ public void check(
   }
 
   @Override
-  public IRowMeta getTableFields(IVariables variables) {
+  public IRowMeta getTableFields(IVariables variables) throws HopDatabaseException {
     IRowMeta fields = null;
     DatabaseMeta databaseMeta =
         getParentTransformMeta().getParentPipelineMeta().findDatabase(connection, variables);
@@ -375,10 +375,11 @@ public IRowMeta getTableFields(IVariables variables) {
                 variables, lookup.getSchemaName(), lookup.getTableName());
         fields = db.getTableFields(schemaTable);
 
-      } catch (HopDatabaseException dbe) {
-        logError(
+      } catch (Throwable dbe) {
+        throw new HopDatabaseException(
             BaseMessages.getString(PKG, "DatabaseLookupMeta.ERROR0004.ErrorGettingTableFields")
-                + dbe.getMessage());
+                + dbe.getMessage(),
+            dbe);
       }
     }
     return fields;
@@ -439,7 +440,8 @@ public void analyseImpact(
       }
     } catch (HopException e) {
       throw new HopTransformException(
-          "Unable to get databaseMeta for connection: " + Const.CR + variables.resolve(connection));
+          "Unable to get databaseMeta for connection: " + Const.CR + variables.resolve(connection),
+          e);
     }
   }
 

plugins/transforms/dbimpact/pom.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~       http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.hop</groupId>
+        <artifactId>hop-plugins-transforms</artifactId>
+        <version>2.18.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>hop-transform-dbimpact</artifactId>
+    <packaging>jar</packaging>
+    <name>Hop Plugins Transforms DB Impact</name>
+
+</project>

plugins/transforms/dbimpact/src/assembly/assembly.xml

@@ -0,0 +1,50 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~       http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+
+<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.2.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.2.0 http://maven.apache.org/xsd/assembly-2.2.0.xsd">
+    <id>hop-transform-dbimpact</id>
+    <formats>
+        <format>zip</format>
+    </formats>
+    <baseDirectory>.</baseDirectory>
+    <files>
+        <file>
+            <source>${project.basedir}/src/main/resources/version.xml</source>
+            <outputDirectory>plugins/transforms/dbimpact</outputDirectory>
+            <filtered>true</filtered>
+        </file>
+    </files>
+
+    <fileSets>
+        <fileSet>
+            <directory>${project.basedir}/src/main/samples</directory>
+            <outputDirectory>config/projects/samples/</outputDirectory>
+        </fileSet>
+    </fileSets>
+
+    <dependencySets>
+        <dependencySet>
+            <includes>
+                <include>org.apache.hop:hop-transform-dbimpact:jar</include>
+            </includes>
+            <outputDirectory>plugins/transforms/dbimpact</outputDirectory>
+        </dependencySet>
+    </dependencySets>
+</assembly>