基于hadoop的统计分析.zip

/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.conf; import java.io.BufferedInputStream; import java.io.DataInput; import java.io.DataOutput; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.io.Reader; import java.io.Writer; import java.net.URL; import java.util.ArrayList; import java.util.Collection; import java.util.Enumeration; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.ListIterator; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.StringTokenizer; import java.util.WeakHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerFactory; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.JsonGenerator; import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.w3c.dom.Text; import org.xml.sax.SAXException; /** * Provides access to configuration parameters. * * <h4 id="Resources">Resources</h4> * * <p> * Configurations are specified by resources. A resource contains a set of name/value pairs as XML data. Each resource is named by either a <code>String</code> or by a {@link Path}. If named by a <code>String</code>, * then the classpath is examined for a file with that name. If named by a <code>Path</code>, then the local filesystem is examined directly, without referring to the classpath. * * <p> * Unless explicitly turned off, Hadoop by default specifies two resources, loaded in-order from the classpath: * <ol> * <li><tt><a href="{@docRoot}/../core-default.html">core-default.xml</a> * </tt>: Read-only defaults for hadoop.</li> * <li><tt>core-site.xml</tt>: Site-specific configuration for a given hadoop installation.</li> * </ol> * Applications may add additional resources, which are loaded subsequent to these resources in the order they are added. * * <h4 id="FinalParams">Final Parameters</h4> * * <p> * Configuration parameters may be declared <i>final</i>. Once a resource declares a value final, no subsequently-loaded resource can alter that value. For example, one might define a final parameter with: <tt><pre> * &lt;property&gt; * &lt;name&gt;dfs.client.buffer.dir&lt;/name&gt; * &lt;value&gt;/tmp/hadoop/dfs/client&lt;/value&gt; * <b>&lt;final&gt;true&lt;/final&gt;</b> * &lt;/property&gt;</pre></tt> * * Administrators typically define parameters as final in <tt>core-site.xml</tt> for values that user applications may not alter. * * <h4 id="VariableExpansion">Variable Expansion</h4> * * <p> * Value strings are first processed for <i>variable expansion</i>. The available properties are: * <ol> * <li>Other properties defined in this Configuration; and, if a name is undefined here,</li> * <li>Properties in {@link System#getProperties()}.</li> * </ol> * * <p> * For example, if a configuration resource contains the following property definitions: <tt><pre> * &lt;property&gt; * &lt;name&gt;basedir&lt;/name&gt; * &lt;value&gt;/user/${<i>user.name</i>}&lt;/value&gt; * &lt;/property&gt; * * &lt;property&gt; * &lt;name&gt;tempdir&lt;/name&gt; * &lt;value&gt;${<i>basedir</i>}/tmp&lt;/value&gt; * &lt;/property&gt;</pre></tt> * * When <tt>conf.get("tempdir")</tt> is called, then <tt>${<i>basedir</i>}</tt> will be resolved to another property in this Configuration, while <tt>${<i>user.name</i>}</tt> would then ordinarily be resolved to the * value of the System property with that name. */ public class Configuration implements Iterable<Map.Entry<String, String>>, Writable { private static final Log LOG = LogFactory.getLog(Configuration.class); private boolean quietmode = true; /** * List of configuration resources. */ private ArrayList<Object> resources = new ArrayList<Object>(); /** * List of configuration parameters marked <b>final</b>. */ private Set<String> finalParameters = new HashSet<String>(); private boolean loadDefaults = true; /** * Configuration objects */ private static final WeakHashMap<Configuration, Object> REGISTRY = new WeakHashMap<Configuration, Object>(); /** * List of default Resources. Resources are loaded in the order of the list entries */ private static final CopyOnWriteArrayList<String> defaultResources = new CopyOnWriteArrayList<String>(); /** * Flag to indicate if the storage of resource which updates a key needs to be stored for each key */ private boolean storeResource; /** * Stores the mapping of key to the resource which modifies or loads the key most recently */ private HashMap<String, String> updatingResource; static { // print deprecation warning if hadoop-site.xml is found in classpath ClassLoader cL = Thread.currentThread().getContextClassLoader(); if (cL == null) { cL = Configuration.class.getClassLoader(); } if (cL.getResource("hadoop-site.xml") != null) { LOG.warn("DEPRECATED: hadoop-site.xml found in the classpath. " + "Usage of hadoop-site.xml is deprecated. Instead use core-site.xml, " + "mapred-site.xml and hdfs-site.xml to override properties of " + "core-default.xml, mapred-default.xml and hdfs-default.xml " + "respectively"); } addDefaultResource("core-default.xml"); addDefaultResource("core-site.xml"); } private Properties properties; private Properties overlay; private ClassLoader classLoader; { classLoader = Thread.currentThread().getContextClassLoader(); if (classLoader == null) { classLoader = Configuration.class.getClassLoader(); } } /** A new configuration. */ public Configuration() { this(true); } /** * A new configuration where the behavior of reading from the default resources can be turned off. * * If the parameter {@code loadDefaults} is false, the new instance will not load resources from the default files. * @param loadDefaults specifies whether to load from the default files */ public Configuration(boolean loadDefaults) { this.loadDefaults = loadDefaults; // if (LOG.isDebugE