Details
-
Type:
Bug
-
Status: Resolved
-
Priority:
Major
-
Resolution: Fixed
-
Affects Version/s: 0.14.1
-
Fix Version/s: 0.15.0
-
Component/s: None
-
Labels:None
Description
It looks like the SchemaTool.createOrMigrateSchemaDirectory doesn't handle key strategies that are defined inside the model schema the way that createOrMigrateSchema does. Here's a test:
I brought in kite-data-hbase 0.14.1. Then created this test file in resources/BrokenKite:
{
"type": "record",
"name": "BrokenRecord",
"namespace": "org.kitesdk.data.hbase.avro.entities",
"tables": ["broken"],
"fields": [
{ "name": "keyPart1", "type": "string", "mapping": { "type": "key", "value": "0" } },
{ "name": "field1", "type": "string", "mapping":
{ "type": "column", "value": "meta:field1" }
},
{
"name": "version",
"type": "long",
"default": 0,
"mapping": { "type": "occVersion" }
}
]
}
Then this test (EDITED FOR THE RIGHT TEST):
import static org.junit.Assert.assertEquals; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTablePool; import org.apache.hadoop.hbase.util.Bytes; import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.kitesdk.data.PartitionKey; import org.kitesdk.data.hbase.avro.SpecificAvroDao; import org.kitesdk.data.hbase.avro.entities.BrokenRecord; import org.kitesdk.data.hbase.impl.Dao; import org.kitesdk.data.hbase.impl.SchemaManager; import org.kitesdk.data.hbase.manager.DefaultSchemaManager; import org.kitesdk.data.hbase.testing.HBaseTestUtils; import org.kitesdk.data.hbase.tool.SchemaTool; /** * Test of using schema tool to migrate schemas in a directory. */ public class SchemaToolTest { private static final String tableName = "broken"; private static final String managedTableName = "managed_schemas"; private HTablePool tablePool; @BeforeClass public static void beforeClass() throws Exception { HBaseTestUtils.getMiniCluster(); } @AfterClass public static void afterClass() throws Exception { HBaseTestUtils.util.deleteTable(Bytes.toBytes(tableName)); } @After public void after() throws Exception { tablePool.close(); HBaseTestUtils.util.truncateTable(Bytes.toBytes(tableName)); HBaseTestUtils.util.truncateTable(Bytes.toBytes(managedTableName)); } @Test public void testMigration() throws Exception { tablePool = new HTablePool(HBaseTestUtils.getConf(), 10); SchemaManager manager =new DefaultSchemaManager(tablePool); SchemaTool tool = new SchemaTool(new HBaseAdmin(HBaseTestUtils.getConf()), manager); tool.createOrMigrateSchemaDirectory("classpath:BrokenKite", true); Dao<BrokenRecord> dao = new SpecificAvroDao<BrokenRecord>(tablePool, tableName, "BrokenRecord", manager); BrokenRecord r1 = new BrokenRecord(); r1.setField1("Field 1"); r1.setKeyPart1("KeyPart 1"); dao.put(r1); PartitionKey key = dao.getPartitionStrategy().partitionKey("KeyPart 1"); BrokenRecord r2 = dao.get(key); assertEquals(r1.getField1().toString(), r2.getField1().toString()); assertEquals(r1.getKeyPart1().toString(), r2.getKeyPart1().toString()); } }
I would expect this to work, but it gets..
org.kitesdk.data.DatasetException: No StorageKey Schema For Table: broken
at org.kitesdk.data.hbase.tool.SchemaTool.createOrMigrateSchemaDirectory(SchemaTool.java:161)
at com.cloudera.kite.broken.SchemaToolTest.testMigration(SchemaToolTest.java:58)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
...
If I change the test to load the schema using createOrMigrateSchema, I think it will work because that's what org.kitesdk.data.hbase.avro.ManagedDaoTest does successfully.