YARN-3948. Display Application Priority in RM Web UI.(Sunil G via rohithsharmaks)

(cherry picked from commit b6265d39)

hadoop-yarn-project/CHANGES.txt

@@ -104,6 +104,8 @@ Release 2.8.0 - UNRELEASED
     YARN-3736. Add RMStateStore apis to store and load accepted reservations for
     failover (adhoot via asuresh)
 
+    YARN-3948. Display Application Priority in RM Web UI.(Sunil G via rohithsharmaks)
+
   IMPROVEMENTS
 
     YARN-644. Basic null check is not performed on passed in arguments before

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java

@@ -408,4 +408,17 @@ public abstract void setLogAggregationStatus(
   @Public
   @Unstable
   public abstract void setUnmanagedApp(boolean unmanagedApplication);
+
+  /**
+   * Get priority of the application
+   *
+   * @return Application's priority
+   */
+  @Public
+  @Stable
+  public abstract Priority getPriority();
+
+  @Private
+  @Unstable
+  public abstract void setPriority(Priority priority);
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto

@@ -196,6 +196,7 @@ message ApplicationReportProto {
   repeated string applicationTags = 20;
   optional LogAggregationStatusProto log_aggregation_status = 21;
   optional bool unmanaged_application = 22 [default = false];
+  optional PriorityProto priority = 23;
 }
 
 enum LogAggregationStatusProto {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java

@@ -501,6 +501,8 @@ private int printApplicationReport(String applicationId)
       appReportStr.println(appReport.getUser());
       appReportStr.print("\tQueue : ");
       appReportStr.println(appReport.getQueue());
+      appReportStr.print("\tApplication Priority : ");
+      appReportStr.println(appReport.getPriority());
       appReportStr.print("\tStart-Time : ");
       appReportStr.println(appReport.getStartTime());
       appReportStr.print("\tFinish-Time : ");

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java

@@ -106,6 +106,7 @@ public void testGetApplicationReport() throws Exception {
           FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN",
           null, null, false);
       newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED);
+      newApplicationReport.setPriority(Priority.newInstance(0));
       when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
           newApplicationReport);
       int result = cli.run(new String[] { "application", "-status", applicationId.toString() });
@@ -119,6 +120,7 @@ public void testGetApplicationReport() throws Exception {
       pw.println("\tApplication-Type : YARN");
       pw.println("\tUser : user");
       pw.println("\tQueue : queue");
+      pw.println("\tApplication Priority : 0");
       pw.println("\tStart-Time : 0");
       pw.println("\tFinish-Time : 0");
       pw.println("\tProgress : 53.79%");

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java

@@ -27,6 +27,7 @@
 import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.LogAggregationStatus;
+import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Token;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
@@ -36,6 +37,7 @@
 import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.LogAggregationStatusProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
 
 import com.google.protobuf.TextFormat;
@@ -55,6 +57,7 @@
   private Token clientToAMToken = null;
   private Token amRmToken = null;
   private Set<String> applicationTags = null;
+  private Priority priority = null;
 
   public ApplicationReportPBImpl() {
     builder = ApplicationReportProto.newBuilder();
@@ -484,6 +487,11 @@ private void mergeLocalToBuilder() {
       builder.clearApplicationTags();
       builder.addAllApplicationTags(this.applicationTags);
     }
+    if (this.priority != null
+        && !((PriorityPBImpl) this.priority).getProto().equals(
+            builder.getPriority())) {
+      builder.setPriority(convertToProtoFormat(this.priority));
+    }
   }
 
   private void mergeLocalToProto() {
@@ -551,6 +559,14 @@ private TokenProto convertToProtoFormat(Token t) {
     return ((TokenPBImpl)t).getProto();
   }
 
+  private PriorityPBImpl convertFromProtoFormat(PriorityProto p) {
+    return new PriorityPBImpl(p);
+  }
+
+  private PriorityProto convertToProtoFormat(Priority t) {
+    return ((PriorityPBImpl)t).getProto();
+  }
+
   @Override
   public LogAggregationStatus getLogAggregationStatus() {
     ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
@@ -593,4 +609,25 @@ public void setUnmanagedApp(boolean unmanagedApplication) {
     maybeInitBuilder();
     builder.setUnmanagedApplication(unmanagedApplication);
   }
+
+  @Override
+  public Priority getPriority() {
+    ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
+    if (this.priority != null) {
+      return this.priority;
+    }
+    if (!p.hasPriority()) {
+      return null;
+    }
+    this.priority = convertFromProtoFormat(p.getPriority());
+    return this.priority;
+  }
+
+  @Override
+  public void setPriority(Priority priority) {
+    maybeInitBuilder();
+    if (priority == null)
+      builder.clearPriority();
+    this.priority = priority;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java

@@ -324,7 +324,8 @@ public static ApplicationReport newApplicationReport(
       String url, long startTime, long finishTime,
       FinalApplicationStatus finalStatus,
       ApplicationResourceUsageReport appResources, String origTrackingUrl,
-      float progress, String appType, Token amRmToken, Set<String> tags) {
+      float progress, String appType, Token amRmToken, Set<String> tags,
+      Priority priority) {
     ApplicationReport report = recordFactory
         .newRecordInstance(ApplicationReport.class);
     report.setApplicationId(applicationId);
@@ -347,6 +348,7 @@ public static ApplicationReport newApplicationReport(
     report.setApplicationType(appType);
     report.setAMRMToken(amRmToken);
     report.setApplicationTags(tags);
+    report.setPriority(priority);
     return report;
   }
   

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java

@@ -167,6 +167,7 @@ public ApplicationReport run() throws Exception {
       ._("Application Type:", app.getType())
       ._("Application Tags:",
         app.getApplicationTags() == null ? "" : app.getApplicationTags())
+      ._("Application Priority:", clarifyAppPriority(app.getPriority()))
       ._(
         "YarnApplicationState:",
         app.getAppState() == null ? UNAVAILABLE : clarifyAppState(app
@@ -342,6 +343,10 @@ private String clarifyAppState(YarnApplicationState state) {
     }
   }
 
+  private String clarifyAppPriority(int priority) {
+    return priority + " (Higher Integer value indicates higher priority)";
+  }
+
   private String clairfyAppFinalStatus(FinalApplicationStatus status) {
     if (status == FinalApplicationStatus.UNDEFINED) {
       return "Application has not completed yet.";

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java

@@ -52,13 +52,13 @@ private static String getAppsTableColumnDefs(
       .append("{'sType':'string', 'aTargets': [0]")
       .append(", 'mRender': parseHadoopID }")
       .append("\n, {'sType':'numeric', 'aTargets': " +
-          (isFairSchedulerPage ? "[6, 7]": "[5, 6]"))
+          (isFairSchedulerPage ? "[6, 7]": "[6, 7]"))
       .append(", 'mRender': renderHadoopDate }")
       .append("\n, {'sType':'numeric', bSearchable:false, 'aTargets':");
     if (isFairSchedulerPage) {
       sb.append("[13]");
     } else if (isResourceManager) {
-      sb.append("[12]");
+      sb.append("[13]");
     } else {
       sb.append("[9]");
     }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java

@@ -58,6 +58,7 @@
   protected long finishedTime;
   protected long elapsedTime;
   protected String applicationTags;
+  protected int priority;
   private int allocatedCpuVcores;
   private int allocatedMemoryMB;
   protected boolean unmanagedApplication;
@@ -86,6 +87,10 @@ public AppInfo(ApplicationReport app) {
     finishedTime = app.getFinishTime();
     elapsedTime = Times.elapsed(startedTime, finishedTime);
     finalAppStatus = app.getFinalApplicationStatus();
+    priority = 0;
+    if (app.getPriority() != null) {
+      priority = app.getPriority().getPriority();
+    }
     if (app.getApplicationResourceUsageReport() != null) {
       runningContainers = app.getApplicationResourceUsageReport()
           .getNumUsedContainers();
@@ -194,4 +199,8 @@ public String getApplicationTags() {
   public boolean isUnmanagedApp() {
     return unmanagedApplication;
   }
+
+  public int getPriority() {
+    return priority;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java

@@ -660,10 +660,10 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName,
       ApplicationReport report = BuilderUtils.newApplicationReport(
           this.applicationId, currentApplicationAttemptId, this.user,
           this.queue, this.name, host, rpcPort, clientToAMToken,
-          createApplicationState(), diags,
-          trackingUrl, this.startTime, this.finishTime, finishState,
-          appUsageReport, origTrackingUrl, progress, this.applicationType, 
-              amrmToken, applicationTags);
+          createApplicationState(), diags, trackingUrl, this.startTime,
+          this.finishTime, finishState, appUsageReport, origTrackingUrl,
+          progress, this.applicationType, amrmToken, applicationTags,
+          this.submissionContext.getPriority());
       report.setLogAggregationStatus(logAggregationStatus);
       report.setUnmanagedApp(submissionContext.getUnmanagedAM());
       return report;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java

@@ -56,7 +56,8 @@ protected void renderData(Block html) {
     TBODY<TABLE<Hamlet>> tbody =
         html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User")
           .th(".name", "Name").th(".type", "Application Type")
-          .th(".queue", "Queue").th(".starttime", "StartTime")
+          .th(".queue", "Queue").th(".priority", "Application Priority")
+          .th(".starttime", "StartTime")
           .th(".finishtime", "FinishTime").th(".state", "State")
           .th(".finalstatus", "FinalStatus")
           .th(".runningcontainer", "Running Containers")
@@ -106,7 +107,9 @@ protected void renderData(Block html) {
         .append("\",\"")
         .append(
           StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
-            .getQueue()))).append("\",\"").append(app.getStartedTime())
+             .getQueue()))).append("\",\"").append(String
+             .valueOf(app.getPriority()))
+        .append("\",\"").append(app.getStartedTime())
         .append("\",\"").append(app.getFinishedTime())
         .append("\",\"")
         .append(app.getAppState() == null ? UNAVAILABLE : app.getAppState())

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java

@@ -74,7 +74,8 @@
   protected long clusterId;
   protected String applicationType;
   protected String applicationTags = "";
-  
+  protected int priority;
+
   // these are only allowed if acls allow
   protected long startedTime;
   protected long finishedTime;
@@ -130,6 +131,11 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
       this.user = app.getUser().toString();
       this.name = app.getName().toString();
       this.queue = app.getQueue().toString();
+      this.priority = 0;
+      if (app.getApplicationSubmissionContext().getPriority() != null) {
+        this.priority = app.getApplicationSubmissionContext().getPriority()
+            .getPriority();
+      }
       this.progress = app.getProgress() * 100;
       this.diagnostics = app.getDiagnostics().toString();
       if (diagnostics == null || diagnostics.isEmpty()) {
@@ -328,4 +334,8 @@ public LogAggregationStatus getLogAggregationStatus() {
   public boolean isUnmanagedApp() {
     return unmanagedApplication;
   }
+
+  public int getPriority() {
+    return this.priority;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java

@@ -1290,6 +1290,7 @@ public void verifyAppsXML(NodeList nodes, RMApp app) throws JSONException,
           WebServicesTestUtils.getXmlString(element, "name"),
           WebServicesTestUtils.getXmlString(element, "applicationType"),
           WebServicesTestUtils.getXmlString(element, "queue"),
+          WebServicesTestUtils.getXmlInt(element, "priority"),
           WebServicesTestUtils.getXmlString(element, "state"),
           WebServicesTestUtils.getXmlString(element, "finalStatus"),
           WebServicesTestUtils.getXmlFloat(element, "progress"),
@@ -1316,18 +1317,18 @@ public void verifyAppsXML(NodeList nodes, RMApp app) throws JSONException,
   public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException,
       Exception {
 
-    assertEquals("incorrect number of elements", 29, info.length());
+    assertEquals("incorrect number of elements", 30, info.length());
 
     verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"),
         info.getString("name"), info.getString("applicationType"),
-        info.getString("queue"), info.getString("state"),
-        info.getString("finalStatus"), (float) info.getDouble("progress"),
-        info.getString("trackingUI"), info.getString("diagnostics"),
-        info.getLong("clusterId"), info.getLong("startedTime"),
-        info.getLong("finishedTime"), info.getLong("elapsedTime"),
-        info.getString("amHostHttpAddress"), info.getString("amContainerLogs"),
-        info.getInt("allocatedMB"), info.getInt("allocatedVCores"),
-        info.getInt("runningContainers"), 
+        info.getString("queue"), info.getInt("priority"),
+        info.getString("state"), info.getString("finalStatus"),
+        (float) info.getDouble("progress"), info.getString("trackingUI"),
+        info.getString("diagnostics"), info.getLong("clusterId"),
+        info.getLong("startedTime"), info.getLong("finishedTime"),
+        info.getLong("elapsedTime"), info.getString("amHostHttpAddress"),
+        info.getString("amContainerLogs"), info.getInt("allocatedMB"),
+        info.getInt("allocatedVCores"), info.getInt("runningContainers"),
         info.getInt("preemptedResourceMB"),
         info.getInt("preemptedResourceVCores"),
         info.getInt("numNonAMContainerPreempted"),
@@ -1337,8 +1338,8 @@ public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException,
   }
 
   public void verifyAppInfoGeneric(RMApp app, String id, String user,
-      String name, String applicationType, String queue, String state,
-      String finalStatus, float progress, String trackingUI,
+      String name, String applicationType, String queue, int prioirty,
+      String state, String finalStatus, float progress, String trackingUI,
       String diagnostics, long clusterId, long startedTime, long finishedTime,
       long elapsedTime, String amHostHttpAddress, String amContainerLogs,
       int allocatedMB, int allocatedVCores, int numContainers,
@@ -1355,6 +1356,7 @@ public void verifyAppInfoGeneric(RMApp app, String id, String user,
     WebServicesTestUtils.checkStringMatch("applicationType",
       app.getApplicationType(), applicationType);
     WebServicesTestUtils.checkStringMatch("queue", app.getQueue(), queue);
+    assertEquals("priority doesn't match", 0, prioirty);
     WebServicesTestUtils.checkStringMatch("state", app.getState().toString(),
         state);
     WebServicesTestUtils.checkStringMatch("finalStatus", app

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md

@@ -1382,6 +1382,7 @@ Response Body:
           "memorySeconds" : 151730,
           "vcoreSeconds" : 103,
           "unmanagedApplication":"false"
+          "applicationPriority":0
        },
        {
           "finishedTime" : 1326815789546,
@@ -1408,6 +1409,7 @@ Response Body:
           "memorySeconds" : 640064,
           "vcoreSeconds" : 442
           "unmanagedApplication":"false"
+          "applicationPriority":0
        }
     ]
   }
@@ -1458,6 +1460,7 @@ Response Body:
     <memorySeconds>151730</memorySeconds>
     <vcoreSeconds>103</vcoreSeconds>
     <unmanagedApplication>false</unmanagedApplication>
+    <applicationPriority>0</applicationPriority>
   </app>
   <app>
     <id>application_1326815542473_0002</id>
@@ -1484,6 +1487,7 @@ Response Body:
     <memorySeconds>640064</memorySeconds>
     <vcoreSeconds>442</vcoreSeconds>
     <unmanagedApplication>false</unmanagedApplication>
+    <applicationPriority>0</applicationPriority>
   </app>
 </apps>
 ```
@@ -1644,6 +1648,7 @@ Note that depending on security settings a user might not be able to see all the
 | memorySeconds | long | The amount of memory the application has allocated (megabyte-seconds) |
 | vcoreSeconds | long | The amount of CPU resources the application has allocated (virtual core-seconds) |
 | unmanagedApplication | boolean | Is the application unmanaged. |
+| applicationPriority | int | priority of the submitted application |
 
 ### Response Examples
 
@@ -1685,6 +1690,7 @@ Response Body:
       "memorySeconds" : 151730,
       "vcoreSeconds" : 103,
       "unmanagedApplication":"false"
+      "applicationPriority":0
    }
 }
 ```
@@ -1727,6 +1733,7 @@ Response Body:
   <memorySeconds>151730</memorySeconds>
   <vcoreSeconds>103</vcoreSeconds>
   <unmanagedApplication>false</unmanagedApplication>
+  <applicationPriority>0</applicationPriority>
 </app>
 ```