From 914fd0a7f974d04528eff9d1f7c405d8926bf8c3 Mon Sep 17 00:00:00 2001
From: Dani Rayan <dr@verticalengine.com>
Date: Wed, 5 Jan 2011 17:13:21 -0800
Subject: [PATCH 6/6] FLUME-406: Update HBase Plugin Build to build and run hbase plugin tests

---
 plugins/hbasesink/build.xml                        |  123 ++++++++++++++++++-
 .../cloudera/flume/hbase/TestAttr2HBaseSink.java   |   36 +++++--
 .../com/cloudera/flume/hbase/TestHBaseSink.java    |  121 -------------------
 3 files changed, 144 insertions(+), 136 deletions(-)
 delete mode 100644 plugins/hbasesink/src/javatest/com/cloudera/flume/hbase/TestHBaseSink.java

diff --git a/plugins/hbasesink/build.xml b/plugins/hbasesink/build.xml
index fcf3c8c..b514b7a 100644
--- a/plugins/hbasesink/build.xml
+++ b/plugins/hbasesink/build.xml
@@ -25,11 +25,36 @@
 
 <project name="flume-hbase" default="jar">
   <property name="javac.debug" value="on"/>
+  <property name="javac.optimize" value="on"/>
+  <property name="javac.deprecation" value="off"/>
+  <property name="javac.version" value="1.5"/>
+  <property name="javac.args" value=""/>
+  <property name="javac.args.warnings" value="-Xlint:unchecked"/>
+
   <property name="flume.base" value="../.."/>
+  <property name="build.dir" value="build"/>
+  <property name="build.test" value="${build.dir}/test"/>
+  <property name="build.encoding" value="ISO-8859-1"/>
+
+  <property name="test.src.dir" value="src/javatest"/>
+  <property name="test.lib.dir" value="${flume.base}/libtest"/>
+  <property name="test.build.dir" value="${build.dir}/test"/>
+  <property name="test.generated.dir" value="${test.build.dir}/src"/>
+  <property name="test.build.data" value="${test.build.dir}/data"/>
+  <property name="test.log.dir" value="${test.build.dir}/logs"/>
+  <property name="test.build.classes" value="${test.build.dir}/classes"/>
+  <property name="test.include" value="Test*"/>
+  <property name="test.classpath.id" value="test.classpath"/>
+  <property name="test.output" value="yes"/>
+  <property name="test.timeout" value="900000"/>
+  <property name="test.junit.output.format" value="plain"/>
 
   <path id="classpath">
     <!-- in case we are running in dev env -->
     <pathelement location="${flume.base}/build/classes"/>
+    <fileset dir="${flume.base}">
+      <include name="flume-*.jar" />
+    </fileset>
     <fileset dir="${flume.base}/lib">
       <include name="**/slf4j-*.jar" />
       <include name="**/hadoop-*.jar" />
@@ -38,21 +63,40 @@
     <fileset dir="${flume.base}/plugins/hbasesink/lib">
       <include name="**/*.jar" />
     </fileset>
-    <!-- in case we are running in release env -->
-    <fileset dir="${flume.base}">
-      <include name="flume-*.jar" />
+  </path>
+
+  <!-- the unit test classpath: uses test.src.dir for configuration -->
+  <path id="test.classpath">
+    <pathelement location="${test.build.classes}" />
+    <pathelement location="${test.src.dir}"/>
+    <pathelement location="${build.dir}"/>
+    <fileset dir="${test.lib.dir}">
+      <include name="**/*.jar" />
+      <exclude name="**/excluded/" />
     </fileset>
+    <fileset dir="${flume.base}/lib">
+      <include name="**/*.jar" />
+      <exclude name="**/excluded/" />
+    </fileset>
+    <fileset dir="${flume.base}/plugins/hbasesink">
+      <include name="**/*.jar" />
+      <exclude name="**/excluded/" />
+    </fileset>
+    <fileset dir="${flume.base}/plugins/hbasesink/lib">
+      <include name="**/*.jar" />
+    </fileset>
+    <path refid="classpath"/>
   </path>
 
   <target name="jar">
-    <mkdir dir="build"/>
-    <mkdir dir="build/classes"/>
+    <mkdir dir="${build.dir}"/>
+    <mkdir dir="${build.dir}/classes"/>
 
-    <javac srcdir="./src/java" destdir="build/classes" debug="${javac.debug}">
+    <javac srcdir="./src/java" destdir="${build.dir}/classes" debug="${javac.debug}">
       <classpath refid="classpath"/>
     </javac>
 
-    <jar jarfile="hbase_sink.jar" basedir="build/classes"/>
+    <jar jarfile="hbase_sink.jar" basedir="${build.dir}/classes"/>
   </target>
 
   <target name="clean">
@@ -61,4 +105,69 @@
     <delete file="hbase_sink.jar" />
   </target>
 
+  <!-- ================================================================== -->
+  <!-- Compile test code                                                  -->
+  <!-- ================================================================== -->
+  <target name="compile-plugin-test" depends="jar">
+    <mkdir dir="${build.dir}/test"/>
+    <mkdir dir="${build.dir}/test/classes"/>
+
+    <javac
+     encoding="${build.encoding}"
+     srcdir="${test.src.dir}"
+     includes="**/*.java"
+     destdir="${test.build.classes}"
+     debug="${javac.debug}"
+     optimize="${javac.optimize}"
+     target="${javac.version}"
+     source="${javac.version}"
+     deprecation="${javac.deprecation}">
+      <compilerarg line="${javac.args}" />
+      <classpath refid="test.classpath"/>
+    </javac>
+ </target>
+
+
+  <!-- ================================================================== -->
+  <!-- Run unit tests                                                     -->
+  <!-- ================================================================== -->
+  <target name="test" depends="jar,compile-plugin-test" >
+    <echo message="Unit Testing of HBase Sink"/>
+    <junit
+      printsummary="yes" showoutput="${test.output}"
+      haltonfailure="no" fork="yes" maxmemory="1024m"
+      errorProperty="tests.failed" failureProperty="tests.failed"
+      timeout="${test.timeout}"
+      dir="${test.build.dir}">
+
+      <!-- uncomment this if you want to attach a debugger -->
+      <!--
+      <jvmarg line="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=2601" />
+      -->
+      <env key="FLUME_HOME" value="${basedir}" />
+      <sysproperty key="javax.xml.parsers.DocumentBuilderFactory" value="com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl" />
+      <sysproperty key="java.library.path" value="${lib.dir}" />
+      <sysproperty key="build.test" value="${build.test}"/>
+
+      <!-- we want more log4j output when running unit tests -->
+      <sysproperty key="hadoop.root.logger"
+        value="INFO,console" />
+
+      <!-- tools.jar from Sun JDK also required to invoke javac. -->
+      <classpath>
+        <path refid="test.classpath"/>
+        <pathelement path="${env.JAVA_HOME}/lib/tools.jar" />
+      </classpath>
+      <formatter type="${test.junit.output.format}" />
+      <batchtest todir="${build.test}" unless="testcase">
+        <fileset dir="${test.src.dir}"
+                 includes="**/Test*.java" excludes="**/${test.exclude}.java" />
+      </batchtest>
+      <batchtest todir="${build.test}" if="testcase">
+        <fileset dir="${test.src.dir}" includes="**/${testcase}.java"/>
+      </batchtest>
+    </junit>
+    <fail if="tests.failed">Tests failed!</fail>
+  </target>
+
 </project>
diff --git a/plugins/hbasesink/src/javatest/com/cloudera/flume/hbase/TestAttr2HBaseSink.java b/plugins/hbasesink/src/javatest/com/cloudera/flume/hbase/TestAttr2HBaseSink.java
index f71d238..421ee1f 100644
--- a/plugins/hbasesink/src/javatest/com/cloudera/flume/hbase/TestAttr2HBaseSink.java
+++ b/plugins/hbasesink/src/javatest/com/cloudera/flume/hbase/TestAttr2HBaseSink.java
@@ -68,7 +68,9 @@ public class TestAttr2HBaseSink {
     final String tableSysFamily = "sysFamily";
     final String tableFamily1 = "family1";
     final String tableFamily2 = "family2";
-    final String tableBodyMap = "sysFamily:event";
+    final String tableBody = "sysFamily";
+    final String tableBodyFamily = "event";
+    final Boolean writeBody = true;
 
     // create the table and column family to be used by sink
     HTableDescriptor desc = new HTableDescriptor(tableName);
@@ -79,7 +81,7 @@ public class TestAttr2HBaseSink {
     admin.createTable(desc);
 
     // explicit constructor rather than builder - we want to control the conf
-    EventSink snk = new Attr2HBaseEventSink(tableName, tableSysFamily, tableBodyMap, "2hb_", 0, true, hbaseEnv.conf);
+    EventSink snk = new Attr2HBaseEventSink(tableName, tableSysFamily, writeBody, tableBody, tableBodyFamily, "2hb_", 0, true, hbaseEnv.conf);
     snk.open();
     try {
       Event e1 = new EventImpl("message0".getBytes(), Clock.unixTime(),
@@ -139,7 +141,10 @@ public class TestAttr2HBaseSink {
 
   @Test
   public void testCreatePutWithoutSystemColumnFamily() {
-    Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "", true, "2hb_", 0, true, null);
+    final String tableBody = "sysFamily";
+    final String tableBodyFamily = "event";
+    final Boolean writeBody = true;
+    Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "", writeBody, tableBody, tableBodyFamily, "2hb_", 0, true, null);
 
     Event e = new EventImpl("message".getBytes(), Clock.unixTime(),
         Priority.INFO, 0, "localhost");
@@ -157,7 +162,10 @@ public class TestAttr2HBaseSink {
 
   @Test
   public void testDontWriteBody() {
-    Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "sysFam", false, "2hb_", 0, true, null);
+    final String tableBody = "";
+    final String tableBodyFamily = "";
+    final Boolean writeBody = false;
+    Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "sysFam", writeBody, tableBody, tableBodyFamily, "2hb_", 0, true, null);
 
     Event e = new EventImpl("message".getBytes(), Clock.unixTime(),
         Priority.INFO, 0, "localhost");
@@ -171,7 +179,10 @@ public class TestAttr2HBaseSink {
 
   @Test
   public void testCreatePutWithoutExplicitRowKey() {
-    Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "sysFam", true, "2hb_", 0, true, null);
+    final String tableBody = "sysFamily";
+    final String tableBodyFamily = "event";
+    final Boolean writeBody = true;
+    Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "sysFam", writeBody, tableBody, tableBodyFamily, "2hb_", 0, true, null);
 
     Event e = new EventImpl("message".getBytes(), Clock.unixTime(),
         Priority.INFO, 0, "localhost");
@@ -183,7 +194,10 @@ public class TestAttr2HBaseSink {
 
   @Test
   public void testAddAttributeWithSystemColumnFamSpecified() {
-    Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "sysFam", true, "2hb_", 0, true, null);
+    final String tableBody = "sysFamily";
+    final String tableBodyFamily = "event";
+    final Boolean writeBody = true;
+    Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "sysFam",writeBody, tableBody, tableBodyFamily, "2hb_", 0, true, null);
     byte[] foo = Bytes.toBytes("foo");
 
     Put put = new Put(foo);
@@ -203,7 +217,10 @@ public class TestAttr2HBaseSink {
 
   @Test
   public void testAddAttributeWithoutSystemColumnFam() {
-    Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "", true, "2hb_", 0, true, null);
+    final String tableBody = "sysFamily";
+    final String tableBodyFamily = "event";
+    final Boolean writeBody = true;
+    Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "", writeBody, tableBody, tableBodyFamily, "2hb_", 0, true, null);
     byte[] foo = Bytes.toBytes("foo");
 
     Put put = new Put(foo);
@@ -236,7 +253,10 @@ public class TestAttr2HBaseSink {
 
   @Test
   public void testAddAttributeWithSystemColumnFamSpecifiedAndEmptyPrefix() {
-    Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "sysFam", true, "", 0, true, null);
+    final String tableBody = "sysFamily";
+    final String tableBodyFamily = "event";
+    final Boolean writeBody = true;
+    Attr2HBaseEventSink snk = new Attr2HBaseEventSink("tableName", "sysFam", writeBody, tableBody, tableBodyFamily, "", 0, true, null);
     byte[] foo = Bytes.toBytes("foo");
 
     Put put = new Put(foo);
diff --git a/plugins/hbasesink/src/javatest/com/cloudera/flume/hbase/TestHBaseSink.java b/plugins/hbasesink/src/javatest/com/cloudera/flume/hbase/TestHBaseSink.java
deleted file mode 100644
index 4726be5..0000000
--- a/plugins/hbasesink/src/javatest/com/cloudera/flume/hbase/TestHBaseSink.java
+++ /dev/null
@@ -1,121 +0,0 @@
-// This Cloudera, Inc. source code, including without limit any
-// human-readable computer programming code and associated documentation
-// (together "Source Code"), contains valuable confidential, proprietary
-// and trade secret information of Cloudera and is protected by the laws
-// of the U.S. and other countries. Any use of the Source Code, including
-// without limit any disclosure or reproduction, without the prior
-// written authorization of Cloudera is strictly prohibited.
-//
-// Copyright (c) 2010 Cloudera, Inc.  All rights reserved.
-package com.cloudera.flume.hbase;
-
-import java.io.IOException;
-
-import junit.framework.Assert;
-
-import org.apache.hadoop.hbase.HBaseTestCase;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import com.cloudera.flume.core.Event;
-import com.cloudera.flume.core.EventImpl;
-import com.cloudera.flume.core.EventSink;
-import com.cloudera.flume.core.Event.Priority;
-import com.cloudera.util.Clock;
-
-/**
- * Test the hbase sink writes events to a table/family properly
- */
-public class TestHBaseSink {
-  private static HBaseTestEnv hbaseEnv;
-  
-  @BeforeClass
-  public static void setup() throws Exception {
-    // expensive, so just do it once for all tests, just make sure
-    // that tests don't overlap (use diff tables for each test)
-    hbaseEnv = new HBaseTestEnv();
-    hbaseEnv.conf.set(HBaseTestCase.TEST_DIRECTORY_KEY, "build/test/data");
-    hbaseEnv.setUp();
-  }
-  
-  @AfterClass
-  public static void teardown() throws Exception {
-    hbaseEnv.tearDown();
-  }
-  
-  /**
-   * Write events to a sink directly, verify by scanning HBase table.
-   */
-  @Test
-  public void testSink() throws IOException {
-    final String tableName = "testSink";
-    final String tableFamily = "testSinkFamily";
-    
-    // create the table and column family to be used by sink
-    HTableDescriptor desc = new HTableDescriptor(tableName);
-    desc.addFamily(new HColumnDescriptor(tableFamily));
-    HBaseAdmin admin = new HBaseAdmin(hbaseEnv.conf);
-    admin.createTable(desc);
-    
-    // explicit constructor rather than builder - we want to control the conf
-    EventSink snk = new HBaseEventSink(tableName, tableFamily, hbaseEnv.conf);
-    snk.open();
-    try {
-      long day_millis = 1000 * 60 * 60 * 24;
-      Event e1 = new EventImpl("message0".getBytes(), Clock.unixTime(),
-          Priority.INFO, 0, "localhost");
-      Event e2 = new EventImpl("message1".getBytes(),
-          e1.getTimestamp() + day_millis, Priority.INFO, 1, "localhost");
-      Event e3 = new EventImpl("message2".getBytes(),
-          e1.getTimestamp() + 2 * day_millis, Priority.INFO, 2, "localhost");
-      snk.append(e1);
-      snk.append(e2);
-      snk.append(e3);
-    } finally {
-      snk.close();
-    }
-    
-    // verify that the events made it into hbase
-    HTable table = new HTable(hbaseEnv.conf, tableName);
-    try {
-      for(long i = 0; i <=2; i++) {
-        Result r = table.get(new Get(Bytes.toBytes(i)));
-        System.out.println("result " + r);
-        byte [] value = r.getValue(Bytes.toBytes(tableFamily),
-          Bytes.toBytes("event"));
-        Assert.assertEquals("Matching body", "message" + i, Bytes.toString(value));
-      }
-    } finally {
-      table.close();
-    }
-  }
-
-  /** Useful for debugging purposes. Scan table and dump to stdout */
-  private void dumpByTimestamp(HTable table, String tableFamily, String colName)
-    throws IOException
-  {
-    Scan s = new Scan();
-    s.addColumn(Bytes.toBytes(tableFamily), Bytes.toBytes(colName));
-    ResultScanner scanner = table.getScanner(s);
-    System.out.println("Scanning " + table.toString() + " " + tableFamily);
-    try {
-      for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
-        System.out.println("Found row: " + rr);
-      }
-
-    } finally {
-      scanner.close();
-    }
-  }
-
-}
-- 
1.7.0.4

