commit 58602f5b4c62e7ec99311eb03e40dd60d32111aa Author: Cao Ying Date: Wed Apr 5 05:52:45 2017 -0700 KITE-1151 : Refactor kite to support other hive external table on filesystem than HDFS diff --git a/kite-data/kite-data-hive/src/main/java/org/kitesdk/data/spi/hive/HiveAbstractDatasetRepository.java b/kite-data/kite-data-hive/src/main/java/org/kitesdk/data/spi/hive/HiveAbstractDatasetRepository.java index 4c7bb67d0..7a477c579 100644 --- a/kite-data/kite-data-hive/src/main/java/org/kitesdk/data/spi/hive/HiveAbstractDatasetRepository.java +++ b/kite-data/kite-data-hive/src/main/java/org/kitesdk/data/spi/hive/HiveAbstractDatasetRepository.java @@ -98,12 +98,16 @@ class HiveAbstractDatasetRepository extends FileSystemDatasetRepository { uri.append(":"); } URI rootUri = rootDirectory.toUri(); - uri.append(rootUri.getPath()); - if (rootUri.getHost() != null) { - uri.append("?").append("hdfs:host").append("=").append(rootUri.getHost()); - if (rootUri.getPort() != -1) { - uri.append("&").append("hdfs:port").append("=").append(rootUri.getPort()); - } + + if(rootUri.getScheme().equals("hdfs")) { + if (rootUri.getHost() != null) { + uri.append("?").append("hdfs:host").append("=").append(rootUri.getHost()); + if (rootUri.getPort() != -1) { + uri.append("&").append("hdfs:port").append("=").append(rootUri.getPort()); + } + } + } else{ + uri.append("/").append(rootUri); } } return URI.create(uri.toString()); diff --git a/kite-data/kite-data-hive/src/main/java/org/kitesdk/data/spi/hive/Loader.java b/kite-data/kite-data-hive/src/main/java/org/kitesdk/data/spi/hive/Loader.java index 84b30efc0..86aeceab0 100644 --- a/kite-data/kite-data-hive/src/main/java/org/kitesdk/data/spi/hive/Loader.java +++ b/kite-data/kite-data-hive/src/main/java/org/kitesdk/data/spi/hive/Loader.java @@ -200,11 +200,17 @@ public class Loader implements Loadable { } else { String defaultScheme; try { - defaultScheme = FileSystem.get(conf).getUri().getScheme(); + defaultScheme = FileSystem.get(new URI(match.get("path")),conf).getUri().getScheme(); } catch (IOException e) { throw new DatasetIOException("Cannot determine the default FS", e); } - return new URI(defaultScheme, userInfo, "", UNSPECIFIED_PORT, "/", null, null); + + if(defaultScheme.equals("s3a")||defaultScheme.equals("swift")) + { + return new URI(match.get("path")); + } else { + return new URI(defaultScheme, userInfo, "", UNSPECIFIED_PORT, "/", null, null); + } } } catch (URISyntaxException ex) { throw new DatasetOperationException("Could not build FS URI", ex); @@ -248,4 +254,4 @@ public class Loader implements Loadable { } return null; } -} +}