/**
* <pre>
* 作 者:CHENYUN
* 创建日期:2021-10-15
* </pre>
*/
package com.chitu.bigdata.sdp.service;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.chitu.bigdata.sdp.api.bo.SdpFileBO;
import com.chitu.bigdata.sdp.api.domain.*;
import com.chitu.bigdata.sdp.api.enums.*;
import com.chitu.bigdata.sdp.api.model.*;
import com.chitu.bigdata.sdp.api.vo.*;
import com.chitu.bigdata.sdp.config.*;
import com.chitu.bigdata.sdp.constant.*;
import com.chitu.bigdata.sdp.flink.common.conf.ConfigConst;
import com.chitu.bigdata.sdp.flink.common.util.DeflaterUtils;
import com.chitu.bigdata.sdp.flink.common.util.HadoopUtils;
import com.chitu.bigdata.sdp.flink.common.util.PropertiesUtils;
import com.chitu.bigdata.sdp.interceptor.EnvHolder;
import com.chitu.bigdata.sdp.mapper.*;
import com.chitu.bigdata.sdp.service.datasource.DataSourceFactory;
import com.chitu.bigdata.sdp.service.datasource.KafkaDataSource;
import com.chitu.bigdata.sdp.service.notify.EmailNotifyService;
import com.chitu.bigdata.sdp.service.validate.ClusterServiceImpl;
import com.chitu.bigdata.sdp.service.validate.constant.FlinkSQLConstant;
import com.chitu.bigdata.sdp.service.validate.custom.CustomTableEnvironmentImpl;
import com.chitu.bigdata.sdp.service.validate.domain.JobConfigs;
import com.chitu.bigdata.sdp.service.validate.domain.SqlExplainResult;
import com.chitu.bigdata.sdp.service.validate.job.JobManager;
import com.chitu.bigdata.sdp.service.validate.util.SqlUtil;
import com.chitu.bigdata.sdp.utils.*;
import com.chitu.bigdata.sdp.utils.flink.sql.MetaTableTypeFetcher;
import com.chitu.bigdata.sdp.utils.flink.sql.ViewTableMapFetcher;
import com.chitu.cloud.exception.ApplicationException;
import com.chitu.cloud.model.ResponseData;
import com.chitu.cloud.service.GenericService;
import com.chitu.cloud.utils.Context;
import com.chitu.cloud.utils.ContextUtils;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.xiaoleilu.hutool.date.DateUtil;
import com.xiaoleilu.hutool.util.CollectionUtil;
import com.xiaoleilu.hutool.util.PageUtil;
import com.xiaoleilu.hutool.util.StrUtil;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlWith;
import org.apache.commons.collections.CollectionUtils;
import org.apache.flink.calcite.shaded.com.google.common.collect.ImmutableList;
import org.apache.flink.sql.parser.ddl.SqlCreateCatalog;
import org.apache.flink.sql.parser.ddl.SqlCreateTable;
import org.apache.flink.sql.parser.ddl.SqlCreateView;
import org.apache.flink.sql.parser.ddl.SqlTableOption;
import org.apache.flink.sql.parser.dml.RichSqlInsert;
import org.apache.flink.table.operations.Operation;
import org.apache.flink.table.operations.ddl.CreateCatalogOperation;
import org.apache.flink.table.operations.ddl.CreateTableOperation;
import org.apache.hadoop.fs.FileSystem;
import org.assertj.core.util.Lists;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cloud.context.config.annotation.RefreshScope;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;
import scala.collection.JavaConversions;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.sql.Timestamp;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static com.chitu.bigdata.sdp.utils.JobOperationUtils.REDIS_KEY_PREFIX;
/**
* @author chenyun
* @description: TODO
* @date 2021/10/17 14:06
*/
@Service
@Slf4j
@RefreshScope
public class FileService extends GenericService<SdpFile, Long> {
public FileService(@Autowired SdpFileMapper sdpFileMapper) {
super(sdpFileMapper);
}
public SdpFileMapper getMapper() {
return (SdpFileMapper) super.genericMapper;
}
private static final String NAME_REGULAR = "([A-Za-z0-9_\\-])+";
private static final String SQL_CONTENT = "%s\r\n%s";
private static final String SQL_PATTEN = "\\$\\{%s\\}";
private static final String YAML_PRE = "sql.param.%s";
private static final String PARSE_RES = "'password' =(.*),";
private static final String PARSE_RES_2 = "\" password=\"(.*)'";
private static final String HIVE_OPTION = "\\/\\*\\+( )*OPTIONS\\(.*\\*\\/";
private static String LINE_REGEX = "line \\d{1,}";
@Autowired
private EmailNotifyService emailNotifyService;
/**
* 指定存量文件的创建时间. ,格式:yyyyMMddHHmm
*/
@Value("${file.edit.checkCreateTime:202211182130}")
private Long checkCreateTime;
@Autowired
private SdpFileMapper sdpFileMapper;
@Autowired
private SdpJobMapper jobMapper;
@Autowired
private SdpJarMapper jarMapper;
@Autowired
private SdpVersionMapper versionMapper;
@Autowired
private SdpJobInstanceMapper instanceMapper;
@Autowired
private SdpProjectEngineMapper sdpProjectEngineMapper;
@Autowired
private SdpUserMapper sdpUserMapper;
@Autowired
private SdpOperationLogMapper logMapper;
@Autowired
private SdpJobAlertRuleService sdpJobAlertRuleService;
@Autowired
private FlinkConfigProperties flinkConfigProperties;
@Autowired
private SdpMetaTableConfigMapper mtaTableConfigMapper;
@Autowired
private DataSourceFactory dataSourceFactory;
@Autowired
private SdpDataSourceMapper sdpDataSourceMapper;
@Autowired
DataSourceService dataSourceService;
@Autowired
private SdpApproveMapper approveMapper;
@Autowired
private SdpFileExtraMapper fileExtraMapper;
@Autowired
private SdpFolderMapper folderMapper;
@Autowired
private MetaTableConfigService configService;
@Autowired
private SdpMetaTableRelationMapper tableRelationMapper;
@Autowired
SdpMetaTableRelationService sdpMetaTableRelationService;
@Autowired
SdpSysConfigService sdpSysConfigService;
@Autowired
UserService userService;
@Autowired
RedisLocker redisLocker;
@Autowired
EngineService engineService;
@Autowired
private ClusterInfoConfig clusterInfoConfig;
@Autowired
DataSourceConfigProperties dataSourceConfigProperties;
@Autowired
ProjectService projectService;
@Autowired
SdpConfig sdpConfig;
@Autowired
CheckConfigProperties checkConfigProperties;
@Autowired
FileService fileService;
@Autowired
JobService jobService;
private final static String ONLINE = "online";
/**
* 保存点路径
*/
@Value("${flink.common.savepointDir}")
private String savepointDir;
@Value("${clusterSyncLocalDir}")
private String clusterSyncLocalDir;
@Autowired
SdpProjectMapper sdpProjectMapper;
@Autowired
RedisTemplateService redisTemplateService;
@Autowired
KafkaDataSource kafkaDataSource;
@Autowired
JarService jarService;
/**
* 检查点间隔时间
*/
private final static Long CHECKPOINTING_INTERVAL = 5 * 60 * 1000L;
private static String COMMENT_REGEX = "(^\\s{0,}--|\\n\\s{0,}--|\\r\\n\\s{0,}--).{0,}";
private static String HANDLE_META_TABLE_RELATION_MSG = "===handleRelation4Metatable[{}]: {} -> {}";
@Transactional(rollbackFor = Exception.class)
public SdpFile addFile(SdpFileBO fileBO) {
Context context = ContextUtils.get();
SdpFile file = new SdpFile();
BeanUtils.copyProperties(fileBO, file);
//移除空行
String newEtlContent = fileBO.getEtlContent();
// if(!StringUtils.i
没有合适的资源?快使用搜索试试~ 我知道了~
温馨提示
【项目资源】: 包含前端、后端、移动开发、操作系统、人工智能、物联网、信息化管理、数据库、硬件开发、大数据、课程资源、音视频、网站开发等各种技术项目的源码。 包括STM32、ESP8266、PHP、QT、Linux、iOS、C++、Java、python、web、C#、EDA、proteus、RTOS等项目的源码。 【项目质量】: 所有源码都经过严格测试,可以直接运行。 功能在确认正常工作后才上传。 【适用人群】: 适用于希望学习不同技术领域的小白或进阶学习者。 可作为毕设项目、课程设计、大作业、工程实训或初期项目立项。 【附加价值】: 项目具有较高的学习借鉴价值,也可直接拿来修改复刻。 对于有一定基础或热衷于研究的人来说,可以在这些基础代码上进行修改和扩展,实现其他功能。 【沟通交流】: 有任何使用上的问题,欢迎随时与博主沟通,博主会及时解答。 鼓励下载和使用,并欢迎大家互相学习,共同进步。
资源推荐
资源详情
资源评论
收起资源包目录
赤兔实时计算平台是基于 Apache Flink 构建的企业级、一站式、高性能、低门槛大数据实时计算平台,广泛适用于.zip (952个子文件)
.browserslistrc 30B
iconfont.css 6KB
org.apache.flink.streaming.api.functions.sink.filesystem.CustomFileSystemFactory 69B
org.apache.flink.streaming.api.functions.sink.filesystem.CustomFileSystemFactory 69B
org.apache.flink.streaming.api.functions.sink.filesystem.CustomFileSystemFactory 69B
org.apache.flink.streaming.api.functions.sink.filesystem.CustomFileSystemFactory 69B
.env.development 33B
.eslintignore 71B
sdp-loading.gif 259KB
help-black.gif 141KB
noti.gif 141KB
help-white.gif 141KB
.gitignore 573B
.gitignore 290B
index.html 4KB
favicon.ico 4KB
FileService.java 124KB
JobService.java 98KB
SdpDataSourceMappingRelService.java 55KB
ProjectService.java 43KB
KafkaDataSource.java 36KB
EngineService.java 33KB
JobStatusSyncThread.java 30KB
ApproveService.java 29KB
GlobalTestController.java 29KB
JarService.java 28KB
MetaTableConfigService.java 26KB
JobRunResultService.java 23KB
DataSourceService.java 20KB
SdpRuntimeLogService.java 20KB
PrometheusQueryService.java 16KB
MybatisInterceptor.java 16KB
SqlParserUtil.java 16KB
JobImageService.java 15KB
HttpClientUtil.java 15KB
FileVersionService.java 15KB
EnvSQLParser.java 14KB
JobController.java 14KB
KafkaConsumerOffsetFetchJob.java 14KB
FileController.java 14KB
ElasticSearchDataSource.java 14KB
CustomTableResultImpl.java 13KB
PermissionAspect.java 12KB
AbstractRuleIndexMonitor.java 12KB
JdbcDataSource.java 12KB
KubernetesApplicationServiceImpl.java 12KB
KubernetesApplicationServiceImpl.java 12KB
HbaseDataSource.java 11KB
GenericService.java 11KB
HdfsUtils.java 10KB
SdpSysConfigService.java 10KB
RedisScheduledAnnotationBeanPostProcessor.java 10KB
SdpJobAlertRuleService.java 9KB
JobStatusNotifyService.java 9KB
ViewTableMapFetcher.java 9KB
AutoPullTaskJob.java 9KB
HbaseCDCTableMonitorJob.java 9KB
EndToEndDelayMonitorJob.java 9KB
Explainer.java 9KB
JobStatusSyncJob.java 8KB
CustomTableEnvironmentImpl.java 8KB
StrUtils.java 8KB
RedisLeaderElection.java 8KB
ResponseCode.java 8KB
ProjectController.java 8KB
CpSpClearJob.java 7KB
JobOperationUtils.java 7KB
MetaTableTypeFetcher.java 6KB
YarnApplicationServiceImpl.java 6KB
MonitorJobStatusError.java 6KB
UserService.java 6KB
SdpDataSourceMappingRel.java 6KB
MathUtils.java 6KB
UatJobAutoOfflineJob.java 5KB
SqlManager.java 5KB
SdpProJobTableDetail.java 5KB
SdpJobInstance.java 5KB
EngineController.java 5KB
JobInstanceService.java 5KB
SdpRuntimeLog.java 5KB
LocalSqlClient.java 5KB
LocalSqlClient.java 5KB
YarnApplicationServiceImpl.java 5KB
DataSourceOption.java 5KB
AbstractDataSource.java 5KB
MetaTableConfigCheckFactory.java 5KB
KafkaMetaTableConfigCheck.java 5KB
HiveDataSource.java 5KB
JsonKafkaDataFormat.java 5KB
ResponseUtils.java 4KB
RuleAlertNotifyService.java 4KB
RedisTemplateService.java 4KB
CanalJsonKafkaDataFormat.java 4KB
SdpJob.java 4KB
SystemConfiguration.java 4KB
YamlUtils.java 4KB
DataSourceController.java 4KB
FolderService.java 4KB
RedisLocker.java 4KB
TopicDelayMonitor.java 4KB
共 952 条
- 1
- 2
- 3
- 4
- 5
- 6
- 10
资源评论
妄北y
- 粉丝: 1w+
- 资源: 1万+
上传资源 快速赚钱
- 我的内容管理 展开
- 我的资源 快来上传第一个资源
- 我的收益 登录查看自己的收益
- 我的积分 登录查看自己的积分
- 我的C币 登录后查看C币余额
- 我的收藏
- 我的下载
- 下载帮助
安全验证
文档复制为VIP权益,开通VIP直接复制
信息提交成功