Apache Kylin分析型数据仓库 v4.0.3 正式版.zip

/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.kylin.common; import java.io.File; import java.io.IOException; import java.io.Serializable; import java.nio.file.Paths; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; import java.util.SortedSet; import java.util.TimeZone; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.text.StrSubstitutor; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.kylin.common.annotation.ConfigTag; import org.apache.kylin.common.lock.DistributedLockFactory; import org.apache.kylin.common.persistence.HDFSResourceStore; import org.apache.kylin.common.util.ClassUtil; import org.apache.kylin.common.util.CliCommandExecutor; import org.apache.kylin.common.util.FileUtils; import org.apache.kylin.common.util.HadoopUtil; import org.apache.kylin.common.util.ParameterFilter; import org.apache.kylin.common.util.StringUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.kylin.shaded.com.google.common.collect.Lists; import org.apache.kylin.shaded.com.google.common.collect.Maps; import org.apache.kylin.shaded.com.google.common.collect.Sets; /** * An abstract class to encapsulate access to a set of 'properties'. * Subclass can override methods in this class to extend the content of the 'properties', * with some override values for example. */ public abstract class KylinConfigBase implements Serializable { private static final long serialVersionUID = 1L; private static final Logger logger = LoggerFactory.getLogger(KylinConfigBase.class); private static final String FALSE = "false"; private static final String TRUE = "true"; private static final String DEFAULT = "default"; private static final String KYLIN_ENGINE_MR_JOB_JAR = "kylin.engine.mr.job-jar"; private static final String KYLIN_ENGINE_PARQUET_JOB_JAR = "kylin.engine.parquet.job-jar"; private static final String KYLIN_STORAGE_HBASE_COPROCESSOR_LOCAL_JAR = "kylin.storage.hbase.coprocessor-local-jar"; private static final String FILE_SCHEME = "file:"; private static final String MAPRFS_SCHEME = "maprfs:"; /* * DON'T DEFINE CONSTANTS FOR PROPERTY KEYS! * * For 1), no external need to access property keys, all accesses are by public methods. * For 2), it's cumbersome to maintain constants at top and code at bottom. * For 3), key literals usually appear only once. */ private static final Pattern COPROCESSOR_JAR_NAME_PATTERN = Pattern.compile("kylin-coprocessor-(.+)\\.jar"); private static final Pattern JOB_JAR_NAME_PATTERN = Pattern.compile("kylin-job-(.+)\\.jar"); private static final Pattern PARQUET_JOB_JAR_NAME_PATTERN = Pattern.compile("kylin-parquet-job-(.+)\\.jar"); // backward compatibility check happens when properties is loaded or updated static BackwardCompatibilityConfig BCC = new BackwardCompatibilityConfig(); volatile Properties properties = new Properties(); // ============================================================================ private String cachedHdfsWorkingDirectory; private String cachedBigCellDirectory; public KylinConfigBase() { this(new Properties()); } public KylinConfigBase(Properties props) { this.properties = BCC.check(props); } protected KylinConfigBase(Properties props, boolean force) { this.properties = force ? props : BCC.check(props); } public static String getKylinConfHome() { String kylinConfHome = getKylinConfHomeWithoutWarn(); if (StringUtils.isEmpty(kylinConfHome)) { logger.warn("KYLIN_CONF was not set"); } return kylinConfHome; } public static String getKylinConfHomeWithoutWarn() { String kylinConfHome = System.getenv("KYLIN_CONF"); if (StringUtils.isEmpty(kylinConfHome)) { kylinConfHome = System.getProperty("KYLIN_CONF"); } return kylinConfHome; } public static String getKylinHome() { String kylinHome = getKylinHomeWithoutWarn(); if (StringUtils.isEmpty(kylinHome)) { logger.warn("KYLIN_HOME was not set"); } return kylinHome; } public static String getKylinHomeWithoutWarn() { String kylinHome = System.getenv("KYLIN_HOME"); if (StringUtils.isEmpty(kylinHome)) { kylinHome = System.getProperty("KYLIN_HOME"); } return kylinHome; } public static String getSparkHome() { String sparkHome = System.getenv("SPARK_HOME"); if (StringUtils.isNotEmpty(sparkHome)) { logger.info("SPARK_HOME was set to {}", sparkHome); return sparkHome; } sparkHome = System.getProperty("SPARK_HOME"); if (StringUtils.isNotEmpty(sparkHome)) { logger.info("SPARK_HOME was set to {}", sparkHome); return sparkHome; } return getKylinHome() + File.separator + "spark"; } private static String getFileName(String homePath, Pattern pattern) { File home = new File(homePath); SortedSet<String> files = Sets.newTreeSet(); if (home.exists() && home.isDirectory()) { File[] listFiles = home.listFiles(); if (listFiles != null) { for (File file : listFiles) { final Matcher matcher = pattern.matcher(file.getName()); if (matcher.matches()) { files.add(file.getAbsolutePath()); } } } } if (files.isEmpty()) { throw new RuntimeException("cannot find " + pattern.toString() + " in " + homePath); } else { return files.last(); } } protected final String getOptional(String prop) { return getOptional(prop, null); } protected String getOptional(String prop, String dft) { final String property = System.getProperty(prop); return property != null ? getSubstitutor().replace(property, System.getenv()) : getSubstitutor().replace(properties.getProperty(prop, dft), System.getenv()); } protected Properties getAllProperties() { return getProperties(null); } /** * @param propertyKeys the collection of the properties; if null will return all properties * @return properties which contained in propertyKeys */ protected Properties getProperties(Collection<String> propertyKeys) { final StrSubstitutor sub = getSubstitutor(); Properties filteredProperties = new Properties(); for (Entry<Object, Object> entry : this.properties.entrySet()) { if (propertyKeys == null || propertyKeys.contains(entry.getKey())) { filteredProperties.put(entry.getKey(), sub.replace((String) entry.getValue())); } } return filteredPrope