diff --git a/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index 1975385..a1e4cb3 100644 --- a/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ b/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -40,7 +40,8 @@ import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; import org.apache.hadoop.mapreduce.util.HostUtil; import org.apache.hadoop.util.Progressable; - +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; /** * Implemention of shims against Hadoop 0.23.0. */ @@ -48,21 +49,50 @@ @Override public String getTaskAttemptLogUrl(JobConf conf, - String taskTrackerHttpAddress, String taskAttemptId) - throws MalformedURLException { - if (conf.get("mapreduce.framework.name") != null - && conf.get("mapreduce.framework.name").equals("yarn")) { - // if the cluster is running in MR2 mode, return null - LOG.warn("Can't fetch tasklog: TaskLogServlet is not supported in MR2 mode."); - return null; - } else { - // if the cluster is running in MR1 mode, using HostUtil to construct TaskLogURL - URL taskTrackerHttpURL = new URL(taskTrackerHttpAddress); - return HostUtil.getTaskLogUrl(taskTrackerHttpURL.getHost(), - Integer.toString(taskTrackerHttpURL.getPort()), - taskAttemptId); - } - } + String taskTrackerHttpAddress, String taskAttemptId) + throws MalformedURLException { + if (isMR2(conf)) { + // if the cluster is running in MR2 mode, return null + LOG.warn( + "Can't fetch tasklog: TaskLogServlet is not supported in MR2 mode."); + + return null; + } else { + // MR2 doesn't have TaskLogServlet class, so need to + String taskLogURL = null; + + try { + Class taskLogClass = Class.forName("TaskLogServlet"); + Method taskLogMethod = taskLogClass.getDeclaredMethod("getTaskLogUrl", + String.class, String.class, String.class); + URL taskTrackerHttpURL = new URL(taskTrackerHttpAddress); + taskLogURL = (String) taskLogMethod.invoke(null, + taskTrackerHttpURL.getHost(), + Integer.toString(taskTrackerHttpURL.getPort()), + taskAttemptId); + } catch (IllegalArgumentException e) { + throw new MalformedURLException( + "Could not execute getTaskLogUrl " + e.getCause()); + } catch (IllegalAccessException e) { + throw new MalformedURLException( + "Could not execute getTaskLogUrl " + e.getCause()); + } catch (InvocationTargetException e) { + throw new MalformedURLException( + "Could not execute getTaskLogUrl " + e.getCause()); + } catch (SecurityException e) { + throw new MalformedURLException( + "Could not execute getTaskLogUrl " + e.getCause()); + } catch (NoSuchMethodException e) { + throw new MalformedURLException( + "Method getTaskLogUrl not found " + e.getCause()); + } catch (ClassNotFoundException e) { + LOG.warn( + "Can't fetch tasklog: TaskLogServlet is not supported in MR2 mode."); + } + + return taskLogURL; + } + } @Override public JobTrackerState getJobTrackerState(ClusterStatus clusterStatus) throws Exception { @@ -95,30 +125,52 @@ @Override public boolean isLocalMode(Configuration conf) { - return "local".equals(conf.get("mapreduce.framework.name")); + if (isMR2(conf)) { + return "local".equals(conf.get("mapreduce.framework.name")); + } + + return "local".equals(conf.get("mapred.job.tracker")); } @Override public String getJobLauncherRpcAddress(Configuration conf) { - return conf.get("yarn.resourcemanager.address"); + if (isMR2(conf)) { + return conf.get("yarn.resourcemanager.address"); + } else { + return conf.get("mapred.job.tracker"); + } } @Override public void setJobLauncherRpcAddress(Configuration conf, String val) { - if (val.equals("local")) { - // LocalClientProtocolProvider expects both parameters to be 'local'. - conf.set("mapreduce.framework.name", val); - conf.set("mapreduce.jobtracker.address", val); - } - else { - conf.set("mapreduce.framework.name", "yarn"); - conf.set("yarn.resourcemanager.address", val); - } + if (val.equals("local")) { + // LocalClientProtocolProvider expects both parameters to be 'local'. + if (isMR2(conf)) { + conf.set("mapreduce.framework.name", val); + conf.set("mapreduce.jobtracker.address", val); + } else { + conf.set("mapred.job.tracker", val); + } + } else { + if (isMR2(conf)) { + conf.set("yarn.resourcemanager.address", val); + } else { + conf.set("mapred.job.tracker", val); + } + } } @Override public String getJobLauncherHttpAddress(Configuration conf) { - return conf.get("yarn.resourcemanager.webapp.address"); + if (isMR2(conf)) { + return conf.get("yarn.resourcemanager.webapp.address"); + } else { + return conf.get("mapred.job.tracker.http.address"); + } + } + + private boolean isMR2(Configuration conf) { + return "yarn".equals(conf.get("mapreduce.framework.name")); } @Override