alluxio.zip

/* * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 * (the "License"). You may not use this work except in compliance with the License, which is * available at www.apache.org/licenses/LICENSE-2.0 * * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, * either express or implied, as more fully set forth in the License. * * See the NOTICE file distributed with this work for information regarding copyright ownership. */ package alluxio.conf; import static alluxio.conf.PropertyKey.Builder.booleanBuilder; import static alluxio.conf.PropertyKey.Builder.classBuilder; import static alluxio.conf.PropertyKey.Builder.dataSizeBuilder; import static alluxio.conf.PropertyKey.Builder.doubleBuilder; import static alluxio.conf.PropertyKey.Builder.durationBuilder; import static alluxio.conf.PropertyKey.Builder.enumBuilder; import static alluxio.conf.PropertyKey.Builder.intBuilder; import static alluxio.conf.PropertyKey.Builder.listBuilder; import static alluxio.conf.PropertyKey.Builder.longBuilder; import static alluxio.conf.PropertyKey.Builder.stringBuilder; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static java.lang.String.format; import alluxio.Constants; import alluxio.DefaultSupplier; import alluxio.ProjectConstants; import alluxio.RuntimeConstants; import alluxio.annotation.PublicApi; import alluxio.client.ReadType; import alluxio.client.WriteType; import alluxio.client.file.cache.ShadowCacheType; import alluxio.client.file.cache.store.PageStoreType; import alluxio.exception.ExceptionMessage; import alluxio.executor.RpcExecutorType; import alluxio.executor.ThreadPoolExecutorQueueType; import alluxio.grpc.LoadMetadataPType; import alluxio.grpc.Scope; import alluxio.grpc.TtlAction; import alluxio.grpc.WritePType; import alluxio.master.GraceMode; import alluxio.master.ZookeeperConnectionErrorPolicy; import alluxio.master.file.MetadataSyncTraversalOrder; import alluxio.master.journal.JournalType; import alluxio.master.metastore.MetastoreType; import alluxio.master.metastore.rocks.DataBlockIndexType; import alluxio.master.metastore.rocks.IndexType; import alluxio.network.ChannelType; import alluxio.security.authentication.AuthType; import alluxio.util.FormatUtils; import alluxio.util.OSUtils; import alluxio.util.compression.DirectoryMarshaller; import alluxio.util.io.PathUtils; import alluxio.worker.block.BlockStoreType; import alluxio.worker.block.management.BackoffStrategy; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.MoreObjects; import com.google.common.base.Objects; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.collect.ImmutableList; import com.sun.management.OperatingSystemMXBean; import io.netty.util.ResourceLeakDetector; import org.rocksdb.CompressionType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.lang.management.ManagementFactory; import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.annotation.Nullable; import javax.annotation.concurrent.ThreadSafe; /** * Configuration property keys. This class provides a set of pre-defined property keys. */ @ThreadSafe @PublicApi public final class PropertyKey implements Comparable<PropertyKey> { /** Regex string to find "${key}" for variable substitution. */ public static final String REGEX_STRING = "(\\$\\{([^{}]*)\\})"; /** Regex to find ${key} for variable substitution. */ public static final Pattern CONF_REGEX = Pattern.compile(REGEX_STRING); private static final Logger LOG = LoggerFactory.getLogger(PropertyKey.class); // The following two maps must be the first to initialize within this file. /** A map from default property key's string name to the key. */ private static final Map<String, PropertyKey> DEFAULT_KEYS_MAP = new ConcurrentHashMap<>(); /** A map from default property key's alias to the key. */ private static final Map<String, PropertyKey> DEFAULT_ALIAS_MAP = new ConcurrentHashMap<>(); /** A cache storing result for template regexp matching results. */ private static final Cache<String, Boolean> REGEXP_CACHE = CacheBuilder.newBuilder() .maximumSize(1024) .build(); /** * The consistency check level to apply to a certain property key. * User can run "alluxio validateEnv all cluster.conf.consistent" to validate the consistency of * configuration properties within the cluster. The command reads all Alluxio properties on * each node of the cluster and shows error or warning for properties that have inconsistent * values among the nodes. For example, it would show error if the Alluxio property for aws secret * access key has some value on a master node and a different value on one of the worker nodes. */ public enum ConsistencyCheckLevel { /** * Do not check consistency of property value. * This should be set if the property is not required to have same value among the nodes * (e.g. worker hostname). */ IGNORE, /** * Check consistency of property value, show warning of inconsistent values. * This should be set if the property is recommended to have same value among the nodes, * although having different values does not cause immediate issues(e.g. alluxio home folder * location, timeout value for connecting to a worker). */ WARN, /** * Check consistency of property value, show error of inconsistent values. * This should be set if the property is required to have same value among the nodes * (e.g. AWS credentials, journal location). */ ENFORCE, } /** * Indicates how the property value should be displayed. */ public enum DisplayType { /** * The property value should be displayed normally. */ DEFAULT, /** * The property value contains credentials and should not be displayed directly. */ CREDENTIALS, } /** * Indicates property type. */ public enum PropertyType { /** * The Property's value is of boolean type, stored as a Boolean. */ BOOLEAN(Boolean.class), /** * The Property's value is of integer type, stored as an Integer. */ INTEGER(Integer.class), /** * The Property's value is of long integer type, stored as a long. */ LONG(Long.class), /** * The Property's value is of double type, stored as a Double. */ DOUBLE(Double.class), /** * The Property's value is of string type, stored as a String. */ STRING(String.class), /** * The Property's value represents a time duration, stored as a Long in ms. */ DURATION(String.class), /** * The Property's value represents a data size, stored as a Long in bytes. */ DATASIZE(String.class), /** * The Property's value is of list type, stored as a delimiter separated string. */ LIST(String.class), /** * The Property's value is an enum for a predefined enum class. */ ENUM(Enum.class), /** * The Property's value is of class type, stored as a Class. */ CLASS(String.class); private final Class<?> mJavaType; PropertyType(Class<?> javaType) { mJavaType = javaType; } Class<?> getJavaType() { return mJavaType; } } public static final Function<Object, Boolean> CHECK_FILE_EXISTS = (fileName) -> { if (!(fileN