diff --git a/core/src/main/java/org/pentaho/di/ui/verticabulkload/VerticaBulkLoaderDialog.java b/core/src/main/java/org/pentaho/di/ui/verticabulkload/VerticaBulkLoaderDialog.java index 9cc7ea7f4..7d82dc600 100644 --- a/core/src/main/java/org/pentaho/di/ui/verticabulkload/VerticaBulkLoaderDialog.java +++ b/core/src/main/java/org/pentaho/di/ui/verticabulkload/VerticaBulkLoaderDialog.java @@ -124,6 +124,10 @@ public class VerticaBulkLoaderDialog extends BaseStepDialog implements StepDialo private Button wDirect; private FormData fdlDirect, fdDirect; + private Label wlEnforcelength; + private Button wEnforcelength; + private FormData fdlEnforcelength, fdEnforcelength; + private Label wlSpecifyFields; private Button wSpecifyFields; private FormData fdlSpecifyFields, fdSpecifyFields; @@ -344,6 +348,26 @@ public void widgetSelected( SelectionEvent arg0 ) { wDirect.setLayoutData( fdDirect ); wDirect.addSelectionListener( lsSelMod ); + // Checkbox for enforcelength + wlEnforcelength = new Label( wMainComp, SWT.RIGHT ); + wlEnforcelength.setText( BaseMessages.getString( PKG, "VerticaBulkLoaderDialog.Enforcelength.Label" ) ); + wlEnforcelength.setToolTipText( BaseMessages.getString( PKG, "VerticaBulkLoaderDialog.Enforcelength.Tooltip" ) ); + props.setLook( wlEnforcelength ); + fdlEnforcelength = new FormData(); + fdlEnforcelength.left = new FormAttachment( 0, 0 ); + fdlEnforcelength.top = new FormAttachment( wDirect, margin ); + fdlEnforcelength.right = new FormAttachment( middle, -margin ); + wlEnforcelength.setLayoutData( fdlEnforcelength ); + wEnforcelength = new Button( wMainComp, SWT.CHECK ); + wEnforcelength.setToolTipText( BaseMessages.getString( PKG, "VerticaBulkLoaderDialog.Enforcelength.Tooltip" ) ); + props.setLook( wEnforcelength ); + fdEnforcelength = new FormData(); + fdEnforcelength.left = new FormAttachment( middle, 0 ); + fdEnforcelength.top = new FormAttachment( wDirect, margin ); + fdEnforcelength.right = new FormAttachment( 100, 0 ); + wEnforcelength.setLayoutData( fdEnforcelength ); + wEnforcelength.addSelectionListener( lsSelMod ); + // Abort on error wlAbortOnError = new Label( wMainComp, SWT.RIGHT ); wlAbortOnError.setText( BaseMessages.getString( PKG, "VerticaBulkLoaderDialog.AbortOnError.Label" ) ); @@ -351,7 +375,7 @@ public void widgetSelected( SelectionEvent arg0 ) { props.setLook( wlAbortOnError ); fdlAbortOnError = new FormData(); fdlAbortOnError.left = new FormAttachment( 0, 0 ); - fdlAbortOnError.top = new FormAttachment( wDirect, margin ); + fdlAbortOnError.top = new FormAttachment( wEnforcelength, margin ); fdlAbortOnError.right = new FormAttachment( middle, -margin ); wlAbortOnError.setLayoutData( fdlAbortOnError ); wAbortOnError = new Button( wMainComp, SWT.CHECK ); @@ -359,7 +383,7 @@ public void widgetSelected( SelectionEvent arg0 ) { props.setLook( wAbortOnError ); fdAbortOnError = new FormData(); fdAbortOnError.left = new FormAttachment( middle, 0 ); - fdAbortOnError.top = new FormAttachment( wDirect, margin ); + fdAbortOnError.top = new FormAttachment( wEnforcelength, margin ); fdAbortOnError.right = new FormAttachment( 100, 0 ); wAbortOnError.setLayoutData( fdAbortOnError ); wAbortOnError.addSelectionListener( lsSelMod ); @@ -829,6 +853,7 @@ public void getData() { } wDirect.setSelection( input.isDirect() ); + wEnforcelength.setSelection( input.isEnforcelength() ); wAbortOnError.setSelection( input.isAbortOnError() ); wSpecifyFields.setSelection( input.specifyFields() ); @@ -864,6 +889,7 @@ private void getInfo( VerticaBulkLoaderMeta info ) { info.setStreamName( wStreamName.getText() ); info.setDirect( wDirect.getSelection() ); + info.setEnforcelength( wEnforcelength.getSelection() ); info.setAbortOnError( wAbortOnError.getSelection() ); info.setSpecifyFields( wSpecifyFields.getSelection() ); diff --git a/core/src/main/java/org/pentaho/di/verticabulkload/VerticaBulkLoader.java b/core/src/main/java/org/pentaho/di/verticabulkload/VerticaBulkLoader.java index 38ce55745..a708f2b9f 100644 --- a/core/src/main/java/org/pentaho/di/verticabulkload/VerticaBulkLoader.java +++ b/core/src/main/java/org/pentaho/di/verticabulkload/VerticaBulkLoader.java @@ -403,7 +403,9 @@ private String buildCopyStatementSqlString() { } // TODO: Should eventually get a preference for this, but for now, be backward compatible. - sb.append( "ENFORCELENGTH " ); + if ( !meta.isEnforcelength() ) { + sb.append("ENFORCELENGTH "); + } if ( meta.isAbortOnError() ) { sb.append( "ABORT ON ERROR " ); diff --git a/core/src/main/java/org/pentaho/di/verticabulkload/VerticaBulkLoaderMeta.java b/core/src/main/java/org/pentaho/di/verticabulkload/VerticaBulkLoaderMeta.java index 935caf491..be7a76904 100644 --- a/core/src/main/java/org/pentaho/di/verticabulkload/VerticaBulkLoaderMeta.java +++ b/core/src/main/java/org/pentaho/di/verticabulkload/VerticaBulkLoaderMeta.java @@ -80,6 +80,9 @@ public class VerticaBulkLoaderMeta extends BaseStepMeta implements StepMetaInter @Injection( name = "DIRECT", group = "MAIN_OPTIONS" ) private boolean direct = true; + @Injection( name = "ENFORCELENGTH", group = "MAIN_OPTIONS" ) + private boolean enforcelength = true; + @Injection( name = "ABORTONERROR", group = "MAIN_OPTIONS" ) private boolean abortOnError = true; @@ -205,6 +208,12 @@ public void setDirect( boolean direct ) { this.direct = direct; } + public boolean isEnforcelength() { return enforcelength; } + + public void setEnforcelength( boolean enforcelength ) { + this.enforcelength = enforcelength; + } + public boolean isAbortOnError() { return abortOnError; } @@ -268,6 +277,7 @@ private void readData( Node stepnode, List data rejectedDataFileName = XMLHandler.getTagValue( stepnode, "rejected_data_filename" ); abortOnError = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "abort_on_error" ) ); direct = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "direct" ) ); + enforcelength = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "enforcelength" ) ); streamName = XMLHandler.getTagValue( stepnode, "stream_name" ); } catch ( Exception e ) { @@ -305,6 +315,7 @@ public String getXML() { retval.append( " " + XMLHandler.addTagValue( "rejected_data_filename", rejectedDataFileName ) ); retval.append( " " + XMLHandler.addTagValue( "abort_on_error", abortOnError ) ); retval.append( " " + XMLHandler.addTagValue( "direct", direct ) ); + retval.append( " " + XMLHandler.addTagValue( "enforcelength", enforcelength ) ); retval.append( " " + XMLHandler.addTagValue( "stream_name", streamName ) ); return retval.toString(); @@ -334,6 +345,7 @@ public void readRep( Repository rep, ObjectId id_step, List databa rejectedDataFileName = rep.getStepAttributeString( id_step, "rejected_data_filename" ); abortOnError = rep.getStepAttributeBoolean( id_step, "abort_on_error" ); direct = rep.getStepAttributeBoolean( id_step, "direct" ); + enforcelength = rep.getStepAttributeBoolean( id_step, "enforcelength" ); streamName = rep.getStepAttributeString( id_step, "stream_name" ); } catch ( Exception e ) { throw new KettleException( "Unexpected error reading step information from the repository", e ); @@ -359,6 +371,7 @@ public void saveRep( Repository rep, ObjectId id_transformation, ObjectId id_ste rep.saveStepAttribute( id_transformation, id_step, "rejected_data_filename", rejectedDataFileName ); rep.saveStepAttribute( id_transformation, id_step, "abort_on_error", abortOnError ); rep.saveStepAttribute( id_transformation, id_step, "direct", direct ); + rep.saveStepAttribute( id_transformation, id_step, "enforcelength", enforcelength ); rep.saveStepAttribute( id_transformation, id_step, "stream_name", streamName ); } catch ( Exception e ) { throw new KettleException( "Unable to save step information to the repository for id_step=" + id_step, e ); diff --git a/core/src/main/resources/org/pentaho/di/verticabulkload/messages/messages_en_US.properties b/core/src/main/resources/org/pentaho/di/verticabulkload/messages/messages_en_US.properties index 75875a537..3c86af99e 100644 --- a/core/src/main/resources/org/pentaho/di/verticabulkload/messages/messages_en_US.properties +++ b/core/src/main/resources/org/pentaho/di/verticabulkload/messages/messages_en_US.properties @@ -29,6 +29,8 @@ VerticaBulkLoaderDialog.AbortOnError.Label=Abort on error VerticaBulkLoaderDialog.AbortOnError.Tooltip=If a record is rejected, the statement will be aborted and no data will be loaded. If this option is not enabled, rejected records will be logged but will not stop the bulk load. VerticaBulkLoaderDialog.InsertDirect.Label=Insert direct to ROS VerticaBulkLoaderDialog.InsertDirect.Tooltip=If enabled, the statement is a COPY DIRECT statement and Vertica will insert the data directly to the ROS (Read Optimized Storage). Otherwise, the data will be inserted to the WOS (Write Optimized Storage) (e.g. a "trickle load") +VerticaBulkLoaderDialog.Enforcelength.Label=Enabled Enforcelength +VerticaBulkLoaderDialog.Enforcelength.Tooltip=If enabled, the ENFORCERLENGTH parameter to reject rows that exceed the length of the target table. VerticaBulkLoaderDialog.Delimiter.Label=Delimiter character VerticaBulkLoaderDialog.Delimiter.Tooltip=Specifies the single-character column delimiter used during the load. Default is the tab character (\\t). Be sure to use a character that is not found in any field of the records because Vertica does not use field quoting. VerticaBulkLoaderDialog.NullString.Label=Null string diff --git a/core/src/test/java/org/pentaho/di/verticabulkload/VerticaBulkLoaderMetaInjectionTest.java b/core/src/test/java/org/pentaho/di/verticabulkload/VerticaBulkLoaderMetaInjectionTest.java index eb80f0721..bbf08a536 100644 --- a/core/src/test/java/org/pentaho/di/verticabulkload/VerticaBulkLoaderMetaInjectionTest.java +++ b/core/src/test/java/org/pentaho/di/verticabulkload/VerticaBulkLoaderMetaInjectionTest.java @@ -48,6 +48,11 @@ public boolean get() { return meta.isDirect(); } } ); + check( "ENFORCELENGTH", new BooleanGetter() { + public boolean get() { + return meta.isEnforcelength(); + } + } ); check( "ABORTONERROR", new BooleanGetter() { public boolean get() { return meta.isAbortOnError(); diff --git a/samples/test_enforcelength.ktr b/samples/test_enforcelength.ktr new file mode 100755 index 000000000..788c32cd8 --- /dev/null +++ b/samples/test_enforcelength.ktr @@ -0,0 +1,594 @@ + + + + enforcelength + + + + Normal + 0 + / + + + + + + + + + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + N + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + N + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + N + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + + + + + + + + + + + 2019/05/07 14:31:55.574 + - + 2009/03/07 18:51:31.454 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + VerticaTest + localhost + VERTICA5 + Native + tuenti + 5433 + dbadmin + Encrypted + + + + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + 5433 + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + SQL_CONNECT + SET SESSION CHARACTERISTICS AS TRANSACTION +ISOLATION LEVEL READ COMMITTED + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + N + + + USE_POOLING + N + + + + + + Data grid + Vertica Bulk Loader + Y + + + + Data grid + DataGrid + + Y + + 1 + + none + + + + + text_test + String + + + + + 2 + -1 + N + + + + + 1 + + + aa + + + aaa + + + + + + + + + + + + 144 + 48 + Y + + + + Vertica Bulk Loader + VerticaBulkLoader + + Y + + 1 + + none + + + VerticaTest + +
enforcelength
+ N + + + + + Y + Y + N + enforcelength + + + + + + + + + + 320 + 48 + Y + + + + + + + N + +