mapred.zip_hadoop_hadoop mapreduce_mapReduce

/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.mapred; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintWriter; import java.io.InputStreamReader; import java.io.Writer; import java.lang.management.ManagementFactory; import java.net.BindException; import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.security.PrivilegedExceptionAction; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; import java.util.Vector; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.mapred.JobSubmissionProtocol; import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenSecretManager; import org.apache.hadoop.http.HttpServer; import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.ipc.RPC.VersionMismatch; import org.apache.hadoop.mapred.AuditLogger.Constants; import org.apache.hadoop.mapred.Counters.CountersExceededException; import org.apache.hadoop.mapred.JobHistory.Keys; import org.apache.hadoop.mapred.JobHistory.Listener; import org.apache.hadoop.mapred.JobHistory.Values; import org.apache.hadoop.mapred.JobInProgress.KillInterruptedException; import org.apache.hadoop.mapred.JobStatusChangeEvent.EventType; import org.apache.hadoop.mapred.QueueManager.QueueACL; import org.apache.hadoop.mapred.TaskTrackerStatus.TaskTrackerHealthStatus; import org.apache.hadoop.net.DNSToSwitchMapping; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.Node; import org.apache.hadoop.net.NodeBase; import org.apache.hadoop.net.ScriptBasedMapping; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.Groups; import org.apache.hadoop.security.RefreshUserMappingsProtocol; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol; import org.apache.hadoop.security.authorize.ServiceAuthorizationManager; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.HostsFileReader; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.mapreduce.ClusterMetrics; import org.apache.hadoop.mapreduce.JobSubmissionFiles; import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.mapreduce.security.token.DelegationTokenRenewal; import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager; import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.util.MBeans; import org.apache.hadoop.security.Credentials; import org.mortbay.util.ajax.JSON; /******************************************************* * JobTracker is the central location for submitting and * tracking MR jobs in a network environment. * *******************************************************/ public class JobTracker implements MRConstants, InterTrackerProtocol, JobSubmissionProtocol, TaskTrackerManager, RefreshUserMappingsProtocol, RefreshAuthorizationPolicyProtocol, AdminOperationsProtocol, JobTrackerMXBean { static{ Configuration.addDefaultResource("mapred-default.xml"); Configuration.addDefaultResource("mapred-site.xml"); } static long TASKTRACKER_EXPIRY_INTERVAL = 10 * 60 * 1000; static long RETIRE_JOB_INTERVAL; static long RETIRE_JOB_CHECK_INTERVAL; private final long DELEGATION_TOKEN_GC_INTERVAL = 3600000; // 1 hour private final DelegationTokenSecretManager secretManager; // The maximum fraction (range [0.0-1.0]) of nodes in cluster allowed to be // added to the all-jobs blacklist via heuristics. By default, no more than // 50% of the cluster can be heuristically blacklisted, but the external // node-healthcheck script is not affected by this. private static double MAX_BLACKLIST_FRACTION = 0.5; // A tracker is blacklisted across jobs only if number of faults is more // than X% above the average number of faults (averaged across all nodes // in cluster). X is the blacklist threshold here; 0.3 would correspond // to 130% of the average, for example. private double AVERAGE_BLACKLIST_THRESHOLD = 0.5; // Fault threshold (number occurring within TRACKER_FAULT_TIMEOUT_WINDOW) // to consider a task tracker bad enough to blacklist heuristically. This // is functionally the same as the older "MAX_BLACKLISTS_PER_TRACKER" value. private int TRACKER_FAULT_THRESHOLD; // = 4; // Width of overall fault-tracking sliding window (in minutes). (Default // of 24 hours matches previous "UPDATE_FAULTY_TRACKER_INTERVAL" value that // was used to forgive a single fault if no others occurred in the interval.) private int TRACKER_FAULT_TIMEOUT_WINDOW; // = 180 (3 hours) // Width of a single fault-tracking bucket (in minutes). private int TRACKER_FAULT_BUCKET_WIDTH; // = 15 private long TRACKER_FAULT_BUCKET_WIDTH_MSECS; // Basically TRACKER_FAULT_TIMEOUT_WINDOW / TRACKER_FAULT_BUCKET_WIDTH . private int NUM_FAULT_BUCKETS; /** the maximum allowed size of the jobconf **/ long MAX_JOBCONF_SIZE = 5*1024*1024L; /** the config key for max user jobconf size **/ public static final String MAX_USER_JOBCONF_SIZE_KEY = "mapred.user.jobconf.limit"; // Delegation token related keys public static final String DELEGATION_KEY_UPDATE_INTERVAL_KEY = "mapreduce.cluster.delegation.key.update-interval"; public static final long DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = 24*60*60*1000; // 1 day public static final String DELEGAT