From 6d000c6a2a6082a1a55b56d0913c95a03f753dd3 Mon Sep 17 00:00:00 2001 From: Carlos Queiroz Date: Tue, 5 Apr 2016 20:12:59 +0700 Subject: [PATCH 1/2] Add support for gpdb header. --- .../config/GPFDistSinkOptionsMetadata.java | 11 +++++++++ .../support/ReadableTableFactoryBean.java | 17 +++++++++++--- .../xd/greenplum/support/SqlUtils.java | 8 +++++-- .../LoadReadableTableFactoryBeanTest.java | 23 +++++++++++++++++++ .../LoadReadableTableFactoryBeanTest.xml | 17 ++++++++++++++ modules/sink/gpfdist/config/gpfdist.xml | 1 + 6 files changed, 72 insertions(+), 5 deletions(-) create mode 100644 extensions/spring-xd-extension-gpfdist/src/test/java/org/springframework/xd/greenplum/support/LoadReadableTableFactoryBeanTest.java create mode 100644 extensions/spring-xd-extension-gpfdist/src/test/resources/org/springframework/xd/greenplum/support/LoadReadableTableFactoryBeanTest.xml diff --git a/extensions/spring-xd-extension-gpfdist/src/main/java/org/springframework/xd/greenplum/config/GPFDistSinkOptionsMetadata.java b/extensions/spring-xd-extension-gpfdist/src/main/java/org/springframework/xd/greenplum/config/GPFDistSinkOptionsMetadata.java index 2b08bb313..9b7c72085 100644 --- a/extensions/spring-xd-extension-gpfdist/src/main/java/org/springframework/xd/greenplum/config/GPFDistSinkOptionsMetadata.java +++ b/extensions/spring-xd-extension-gpfdist/src/main/java/org/springframework/xd/greenplum/config/GPFDistSinkOptionsMetadata.java @@ -68,6 +68,9 @@ public class GPFDistSinkOptionsMetadata { private String sqlAfter; + private boolean header = false; + + public int getPort() { return port; } @@ -257,4 +260,12 @@ public void setSqlAfter(String sqlAfter) { this.sqlAfter = sqlAfter; } + @ModuleOption("header ") + public void setHeader(boolean header){ + this.header = header; + } + + public boolean isHeader(){ + return this.header; + } } diff --git a/extensions/spring-xd-extension-gpfdist/src/main/java/org/springframework/xd/greenplum/support/ReadableTableFactoryBean.java b/extensions/spring-xd-extension-gpfdist/src/main/java/org/springframework/xd/greenplum/support/ReadableTableFactoryBean.java index 7a6a8ce2c..ecdc752ad 100644 --- a/extensions/spring-xd-extension-gpfdist/src/main/java/org/springframework/xd/greenplum/support/ReadableTableFactoryBean.java +++ b/extensions/spring-xd-extension-gpfdist/src/main/java/org/springframework/xd/greenplum/support/ReadableTableFactoryBean.java @@ -16,12 +16,12 @@ package org.springframework.xd.greenplum.support; -import java.util.Arrays; -import java.util.List; - import org.springframework.beans.factory.FactoryBean; import org.springframework.beans.factory.InitializingBean; +import java.util.Arrays; +import java.util.List; + /** * @since 1.2 * @author Janne Valkealahti @@ -57,6 +57,8 @@ public class ReadableTableFactoryBean implements FactoryBean, Ini private SegmentRejectType segmentRejectType; + private boolean header; + @Override public void afterPropertiesSet() throws Exception { if (controlFile != null) { @@ -75,6 +77,7 @@ public ReadableTable getObject() throws Exception { w.setLogErrorsInto(logErrorsInto); w.setSegmentRejectLimit(segmentRejectLimit); w.setSegmentRejectType(segmentRejectType); + w.setFormatHeader(header); if (format == Format.TEXT) { Character delim = delimiter != null ? delimiter : Character.valueOf('\t'); @@ -206,4 +209,12 @@ public void setFormat(Format format) { this.format = format; } + public boolean isHeader() { + return header; + } + + public void setHeader(boolean header) { + this.header = header; + } + } diff --git a/extensions/spring-xd-extension-gpfdist/src/main/java/org/springframework/xd/greenplum/support/SqlUtils.java b/extensions/spring-xd-extension-gpfdist/src/main/java/org/springframework/xd/greenplum/support/SqlUtils.java index 6001ac331..1a381fa97 100644 --- a/extensions/spring-xd-extension-gpfdist/src/main/java/org/springframework/xd/greenplum/support/SqlUtils.java +++ b/extensions/spring-xd-extension-gpfdist/src/main/java/org/springframework/xd/greenplum/support/SqlUtils.java @@ -16,10 +16,10 @@ package org.springframework.xd.greenplum.support; -import java.util.List; - import org.springframework.util.StringUtils; +import java.util.List; + public abstract class SqlUtils { public static String createExternalReadableTable(LoadConfiguration config, String prefix, @@ -101,6 +101,10 @@ else if (StringUtils.hasText(externalTable.getColumns())) { buf.append(StringUtils.arrayToCommaDelimitedString(externalTable.getForceQuote())); } + if(externalTable.isFormatHeader()){ + buf.append(" HEADER "); + } + buf.append(" )"); if (externalTable.getEncoding() != null) { diff --git a/extensions/spring-xd-extension-gpfdist/src/test/java/org/springframework/xd/greenplum/support/LoadReadableTableFactoryBeanTest.java b/extensions/spring-xd-extension-gpfdist/src/test/java/org/springframework/xd/greenplum/support/LoadReadableTableFactoryBeanTest.java new file mode 100644 index 000000000..fa427494e --- /dev/null +++ b/extensions/spring-xd-extension-gpfdist/src/test/java/org/springframework/xd/greenplum/support/LoadReadableTableFactoryBeanTest.java @@ -0,0 +1,23 @@ +package org.springframework.xd.greenplum.support; + +import org.junit.Test; +import org.springframework.context.support.ClassPathXmlApplicationContext; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +/** + * Created by cq on 4/4/16. + */ +public class LoadReadableTableFactoryBeanTest { + + @Test + public void testLoadHeaderConfiguration() throws Exception { + ClassPathXmlApplicationContext context = new ClassPathXmlApplicationContext( + "org/springframework/xd/greenplum/support/LoadReadableTableFactoryBeanTest.xml"); + ReadableTableFactoryBean factoryBean = context.getBean("&greenplumReadableTable", ReadableTableFactoryBean.class); + assertThat(factoryBean.isHeader(),is(true)); + context.close(); + + } +} diff --git a/extensions/spring-xd-extension-gpfdist/src/test/resources/org/springframework/xd/greenplum/support/LoadReadableTableFactoryBeanTest.xml b/extensions/spring-xd-extension-gpfdist/src/test/resources/org/springframework/xd/greenplum/support/LoadReadableTableFactoryBeanTest.xml new file mode 100644 index 000000000..5708a9704 --- /dev/null +++ b/extensions/spring-xd-extension-gpfdist/src/test/resources/org/springframework/xd/greenplum/support/LoadReadableTableFactoryBeanTest.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/modules/sink/gpfdist/config/gpfdist.xml b/modules/sink/gpfdist/config/gpfdist.xml index 4f2dc480f..9259c5f54 100644 --- a/modules/sink/gpfdist/config/gpfdist.xml +++ b/modules/sink/gpfdist/config/gpfdist.xml @@ -60,6 +60,7 @@ + From 1743306cd58374e82497406adfe2bf1114e31539 Mon Sep 17 00:00:00 2001 From: Carlos Queiroz Date: Tue, 5 Apr 2016 20:37:36 +0700 Subject: [PATCH 2/2] fix issue with NetworkUtils package name. --- .../xd/greenplum/support/LoadReadableTableFactoryBeanTest.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/spring-xd-extension-gpfdist/src/test/resources/org/springframework/xd/greenplum/support/LoadReadableTableFactoryBeanTest.xml b/extensions/spring-xd-extension-gpfdist/src/test/resources/org/springframework/xd/greenplum/support/LoadReadableTableFactoryBeanTest.xml index 5708a9704..8d4dcf944 100644 --- a/extensions/spring-xd-extension-gpfdist/src/test/resources/org/springframework/xd/greenplum/support/LoadReadableTableFactoryBeanTest.xml +++ b/extensions/spring-xd-extension-gpfdist/src/test/resources/org/springframework/xd/greenplum/support/LoadReadableTableFactoryBeanTest.xml @@ -7,7 +7,7 @@ - +