一个支持多种异构数据源的质量校验、通知、管理服务的数据质量管理平台，用于解决业务系统运行、数据中心建设及数据治理过程中的各种数据

/* * Copyright 2019 WeBank * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.webank.wedatasphere.qualitis.service.impl; import com.alibaba.druid.DbType; import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlSchemaStatVisitor; import com.alibaba.druid.stat.TableStat; import com.alibaba.druid.stat.TableStat.Name; import com.alibaba.druid.util.JdbcConstants; import com.alibaba.excel.support.ExcelTypeEnum; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.Gson; import com.webank.wedatasphere.qualitis.EngineTypeEnum; import com.webank.wedatasphere.qualitis.bean.LogResult; import com.webank.wedatasphere.qualitis.bean.TaskSubmitResult; import com.webank.wedatasphere.qualitis.checkalert.dao.CheckAlertDao; import com.webank.wedatasphere.qualitis.checkalert.dao.repository.CheckAlertWhiteListRepository; import com.webank.wedatasphere.qualitis.checkalert.entity.CheckAlert; import com.webank.wedatasphere.qualitis.checkalert.entity.CheckAlertWhiteList; import com.webank.wedatasphere.qualitis.client.impl.ImsAlarmClient; import com.webank.wedatasphere.qualitis.concurrent.RuleContext; import com.webank.wedatasphere.qualitis.concurrent.RuleContextManager; import com.webank.wedatasphere.qualitis.config.ImsConfig; import com.webank.wedatasphere.qualitis.config.LinkisConfig; import com.webank.wedatasphere.qualitis.config.SystemKeyConfig; import com.webank.wedatasphere.qualitis.constant.*; import com.webank.wedatasphere.qualitis.constants.QualitisConstants; import com.webank.wedatasphere.qualitis.constants.WhiteListTypeEnum; import com.webank.wedatasphere.qualitis.dao.*; import com.webank.wedatasphere.qualitis.dao.repository.TaskDataSourceRepository; import com.webank.wedatasphere.qualitis.dto.SubmitRuleBaseInfo; import com.webank.wedatasphere.qualitis.entity.*; import com.webank.wedatasphere.qualitis.exception.*; import com.webank.wedatasphere.qualitis.job.MonitorManager; import com.webank.wedatasphere.qualitis.metadata.client.MetaDataClient; import com.webank.wedatasphere.qualitis.metadata.constant.RuleConstraintEnum; import com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException; import com.webank.wedatasphere.qualitis.metadata.request.GetUserColumnByCsRequest; import com.webank.wedatasphere.qualitis.metadata.request.GetUserTableByCsIdRequest; import com.webank.wedatasphere.qualitis.metadata.response.DataInfo; import com.webank.wedatasphere.qualitis.metadata.response.column.ColumnInfoDetail; import com.webank.wedatasphere.qualitis.metadata.response.table.CsTableInfoDetail; import com.webank.wedatasphere.qualitis.metadata.response.table.PartitionStatisticsInfo; import com.webank.wedatasphere.qualitis.metadata.response.table.TableStatisticsInfo; import com.webank.wedatasphere.qualitis.parser.HiveSqlParser; import com.webank.wedatasphere.qualitis.parser.LocaleParser; import com.webank.wedatasphere.qualitis.project.constant.ProjectUserPermissionEnum; import com.webank.wedatasphere.qualitis.project.dao.ProjectDao; import com.webank.wedatasphere.qualitis.project.dao.ProjectUserDao; import com.webank.wedatasphere.qualitis.project.entity.Project; import com.webank.wedatasphere.qualitis.project.service.ProjectService; import com.webank.wedatasphere.qualitis.request.*; import com.webank.wedatasphere.qualitis.response.*; import com.webank.wedatasphere.qualitis.rule.constant.GroupTypeEnum; import com.webank.wedatasphere.qualitis.rule.constant.TemplateDataSourceTypeEnum; import com.webank.wedatasphere.qualitis.rule.dao.ExecutionParametersDao; import com.webank.wedatasphere.qualitis.rule.dao.RuleDao; import com.webank.wedatasphere.qualitis.rule.dao.RuleDataSourceDao; import com.webank.wedatasphere.qualitis.rule.dao.RuleGroupDao; import com.webank.wedatasphere.qualitis.rule.entity.*; import com.webank.wedatasphere.qualitis.rule.service.*; import com.webank.wedatasphere.qualitis.rule.util.UnitTransfer; import com.webank.wedatasphere.qualitis.scheduled.constant.RuleTypeEnum; import com.webank.wedatasphere.qualitis.service.OuterExecutionService; import com.webank.wedatasphere.qualitis.service.TaskService; import com.webank.wedatasphere.qualitis.submitter.ExecutionManager; import com.webank.wedatasphere.qualitis.util.*; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang3.time.FastDateFormat; import org.apache.hadoop.hive.ql.parse.ParseException; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.client.ResourceAccessException; import javax.annotation.PostConstruct; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.core.Context; import java.math.BigDecimal; import java.security.SecureRandom; import java.text.SimpleDateFormat; import java.util.*; import java.util.stream.Collectors; /** * @author howeye */ @Service public class OuterExecutionServiceImpl implements OuterExecutionService { @Autowired private ImsAlarmClient imsAlarmClient; @Autowired private ImsConfig imsConfig; @Autowired private AlarmInfoDao alarmInfoDao; @Autowired private CheckAlertDao checkAlertDao; @Autowired private TaskResultDao taskResultDao; @Autowired private TaskDao taskDao; @Autowired private RuleDao ruleDao; @Autowired private UserDao userDao; @Autowired private ProjectDao projectDao; @Autowired private RuleGroupDao ruleGroupDao; @Autowired private ApplicationDao applicationDao; @Autowired private ProjectUserDao projectUserDao; @Autowired private ClusterInfoDao clusterInfoDao; @Autowired private ServiceInfoDao serviceInfoDao; @Autowired private SystemConfigDao systemConfigDao; @Autowired private RuleDataSourceDao ruleDataSourceDao; @Autowired private GatewayJobInfoDao gatewayJobInfoDao; @Autowired private ExecutionParametersDao executionParametersDao; @Autowired private AbnormalDataRecordInfoDao abnormalDataRecordInfoDao; @Autowired private CheckAlertWhiteListRepository checkAlertWhiteListRepository; @Autowired private TaskDataSourceRepository taskDataSourceRepository; @Autowired private TaskRuleSimpleDao taskRuleSimpleDao; @Autowired private RuleMetricDao ruleMetricDao; @Autowired private MonitorManager monitorManager; @Autowired private ExecutionManager executionManager; @Autowired private FpsService fpsService; @Autowired private ProjectService projectService; @Autowired private OuterExecutionService outerExecutionService; @Autowired private SystemKeyConfig systemKeyConfig; @Autowired private MetaDataClient metaDataClient; @Autowired private LinkisConfig linkisConfig; @Autowired private LocaleParser localeParser; @Autowired private RuleDataSourceService ruleDataSourceService; @Autowire