package io.dataease.service.dataset;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.reflect.TypeToken;
import io.dataease.auth.annotation.DeCleaner;
import io.dataease.auth.api.dto.CurrentUserDto;
import io.dataease.commons.constants.*;
import io.dataease.commons.exception.DEException;
import io.dataease.commons.utils.*;
import io.dataease.controller.ResultHolder;
import io.dataease.controller.request.dataset.DataSetExportRequest;
import io.dataease.controller.request.dataset.DataSetGroupRequest;
import io.dataease.controller.request.dataset.DataSetTableRequest;
import io.dataease.controller.request.dataset.DataSetTaskRequest;
import io.dataease.controller.response.DataSetDetail;
import io.dataease.controller.sys.base.BaseGridRequest;
import io.dataease.controller.sys.base.ConditionEntity;
import io.dataease.dto.SysLogDTO;
import io.dataease.dto.dataset.*;
import io.dataease.dto.dataset.union.UnionDTO;
import io.dataease.dto.dataset.union.UnionItemDTO;
import io.dataease.dto.dataset.union.UnionParamDTO;
import io.dataease.exception.DataEaseException;
import io.dataease.ext.ExtDataSetGroupMapper;
import io.dataease.ext.ExtDataSetTableMapper;
import io.dataease.ext.UtilMapper;
import io.dataease.i18n.Translator;
import io.dataease.listener.util.CacheUtils;
import io.dataease.plugins.common.base.domain.*;
import io.dataease.plugins.common.base.mapper.*;
import io.dataease.plugins.common.constants.DatasetType;
import io.dataease.plugins.common.constants.DatasourceTypes;
import io.dataease.plugins.common.constants.DeTypeConstants;
import io.dataease.plugins.common.dto.chart.ChartViewFieldDTO;
import io.dataease.plugins.common.dto.dataset.SqlVariableDetails;
import io.dataease.plugins.common.dto.datasource.DataSourceType;
import io.dataease.plugins.common.dto.datasource.TableField;
import io.dataease.plugins.common.request.datasource.DatasourceRequest;
import io.dataease.plugins.common.request.permission.DataSetRowPermissionsTreeDTO;
import io.dataease.plugins.common.request.permission.DatasetRowPermissionsTreeObj;
import io.dataease.plugins.datasource.provider.Provider;
import io.dataease.plugins.datasource.query.QueryProvider;
import io.dataease.plugins.loader.ClassloaderResponsity;
import io.dataease.plugins.xpack.auth.dto.request.ColumnPermissionItem;
import io.dataease.provider.DDLProvider;
import io.dataease.provider.ProviderFactory;
import io.dataease.provider.datasource.JdbcProvider;
import io.dataease.service.chart.util.ChartDataBuild;
import io.dataease.service.datasource.DatasourceService;
import io.dataease.service.engine.EngineService;
import io.dataease.service.sys.SysAuthService;
import net.sf.jsqlparser.expression.Alias;
import net.sf.jsqlparser.expression.BinaryExpression;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.Parenthesis;
import net.sf.jsqlparser.expression.operators.conditional.AndExpression;
import net.sf.jsqlparser.expression.operators.conditional.OrExpression;
import net.sf.jsqlparser.expression.operators.relational.*;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.statement.Statement;
import net.sf.jsqlparser.statement.select.*;
import net.sf.jsqlparser.util.deparser.ExpressionDeParser;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.hssf.usermodel.HSSFDateUtil;
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.text.MessageFormat;
import java.text.NumberFormat;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* @Author gin
* @Date 2021/2/23 2:54 下午
*/
@Service
@Transactional(rollbackFor = Exception.class)
public class DataSetTableService {
@Resource
private DatasetTableMapper datasetTableMapper;
@Resource
private DatasourceMapper datasourceMapper;
@Resource
private DataSetTableFieldsService dataSetTableFieldsService;
@Resource
private DataSetTableTaskService dataSetTableTaskService;
@Resource
private CommonThreadPool commonThreadPool;
@Resource
private ExtractDataService extractDataService;
@Resource
private ExtDataSetTableMapper extDataSetTableMapper;
@Resource
private DatasetTableIncrementalConfigMapper datasetTableIncrementalConfigMapper;
@Resource
private DataSetTableUnionService dataSetTableUnionService;
@Resource
private QrtzSchedulerStateMapper qrtzSchedulerStateMapper;
@Resource
private DatasetTableTaskLogMapper datasetTableTaskLogMapper;
@Resource
private ExtDataSetGroupMapper extDataSetGroupMapper;
@Resource
private DatasetTableFieldMapper datasetTableFieldMapper;
@Resource
private PermissionService permissionService;
@Resource
private EngineService engineService;
@Resource
private SysAuthService sysAuthService;
@Resource
private ChartViewMapper chartViewMapper;
@Resource
private DataSetTableTaskLogService dataSetTableTaskLogService;
@Resource
private PermissionsTreeService permissionsTreeService;
@Resource
private DatasourceService datasourceService;
@Resource
private DatasetSqlLogMapper datasetSqlLogMapper;
private static boolean isUpdatingDatasetTableStatus = false;
private static final String lastUpdateTime = "${__last_update_time__}";
private static final String currentUpdateTime = "${__current_update_time__}";
public static final String regex = "\\$\\{(.*?)\\}";
private static final String SubstitutedParams = "DATAEASE_PATAMS_BI";
private static final String SubstitutedSql = " 'BI' = 'BI' ";
private static final String SubstitutedSqlVirtualData = " 1 > 2 ";
@Value("${upload.file.path}")
private String path;
private static final Logger logger = LoggerFactory.getLogger(ClassloaderResponsity.class);
@DeCleaner(value = DePermissionType.DATASET, key = "sceneId")
public List<DatasetTable> batchInsert(List<DataSetTableRequest> datasetTable) throws Exception {
// 保存之前校验table名称
checkNames(datasetTable);
List<DatasetTable> list = new ArrayList<>();
for (DataSetTableRequest table : datasetTable) {
list.add(save(table));
// 清理权限缓存
CacheUtils.removeAll(AuthConstants.USER_DATASET_NAME);
CacheUtils.removeAll(AuthConstants.ROLE_DATASET_NAME);
CacheUtils.removeAll(AuthConstants.DEPT_DATASET_NAME);
}
return list;
}
private void extractData(DataSetTableRequest datasetTable) throws Exception {
if (datasetTable.getMode() == 1 && StringUtils.isNotEmpty(datasetTable.getSyncType())
&& datasetTable.getSyncType().equalsIgnoreCase("sync_now")) {
DataSetTaskRequest dataSetTaskRequest = new DataSetTaskRequest();
DatasetTableTask datasetTableTask = new DatasetTableTask();
datasetTableTask.setTableId(datasetTable.getId());
datasetTableTask.setRate(ScheduleType.SIMPLE.toString());
datasetTableTask.setType("all_scope");
datasetTableTask.setName(datasetTable.getName() + " 更新设置-" + System.currentTimeMillis());
没有合适的资源?快使用搜索试试~ 我知道了~
温馨提示
DataEase 是开源的数据可视化分析工具,帮助用户快速分析数据并洞察业务趋势,从而实现业务的改进与优化。DataEase 支持丰富的数据源连接,能够通过拖拉拽方式快速制作图表,并可以方便的与他人分享。 DataEase 的功能包括: 图表展示:支持 PC 端、移动端、大屏; 图表制作:支持丰富的图表类型(Apache ECharts / AntV)、支持拖拉拽方式快速制作仪表板; 数据引擎:支持直连模式、本地模式(基于 Apache Doris / Kettle 实现); 数据连接:支持数据仓库/数据湖、 OLAP 数据库、OLTP 数据库、Excel 数据文件、API 等各种数据源。 DataEase 的优势 开源开放:零门槛,线上快速获取和安装;快速获取用户反馈、按月发布新版本; 简单易用:极易上手,通过鼠标点击和拖拽即可完成分析; 秒级响应:集成 Apache Doris,超大数据量下秒级查询返回延时; 安全分享:支持多种数据分享方式,确保数据安全。
资源推荐
资源详情
资源评论
收起资源包目录
开源的数据可视化分析工具 (2000个子文件)
animate.css 76KB
skin.css 71KB
skin.css 71KB
skin.min.css 60KB
skin.min.css 60KB
uni.css 26KB
skin.mobile.css 24KB
skin.mobile.css 24KB
content.css 24KB
content.inline.css 24KB
content.inline.css 24KB
content.css 23KB
content.min.css 21KB
content.inline.min.css 21KB
content.inline.min.css 21KB
content.min.css 21KB
skin.mobile.min.css 21KB
skin.mobile.min.css 21KB
demo.css 8KB
uniicons.css 8KB
iconfont.css 6KB
m-icon.css 5KB
custom-theme.css 5KB
iconfont.css 3KB
uni-nvue.css 2KB
content.css 2KB
content.css 2KB
content.css 2KB
content.css 2KB
content.min.css 1KB
content.min.css 1KB
content.min.css 1KB
content.min.css 1KB
skin.shadowdom.css 867B
skin.shadowdom.css 867B
skin.shadowdom.min.css 764B
skin.shadowdom.min.css 764B
content.mobile.css 732B
content.mobile.css 732B
content.mobile.min.css 544B
content.mobile.min.css 544B
index.css 46B
demo_index.html 104KB
link.html 1KB
index.html 1KB
nologin.html 865B
nolic.html 784B
index.html 599B
DataSetTableService.java 157KB
ChartViewService.java 114KB
OracleQueryProvider.java 82KB
ExtractDataService.java 78KB
MysqlQueryProvider.java 77KB
Db2QueryProvider.java 75KB
DorisQueryProvider.java 75KB
MysqlQueryProvider.java 74KB
CKQueryProvider.java 74KB
SqlserverQueryProvider.java 74KB
PgQueryProvider.java 71KB
HiveQueryProvider.java 71KB
ImpalaQueryProvider.java 71KB
EsQueryProvider.java 70KB
RedshiftQueryProvider.java 70KB
PanelGroupService.java 60KB
MongoQueryProvider.java 60KB
ChartDataBuild.java 58KB
JdbcProvider.java 44KB
DatasourceService.java 33KB
PanelAppTemplateService.java 26KB
ApiProvider.java 25KB
SysUserService.java 23KB
ShareService.java 20KB
EmailTaskHandler.java 20KB
XEmailTaskServer.java 19KB
ExcelXlsxReader.java 19KB
SystemParameterService.java 19KB
LogService.java 18KB
AuthServer.java 18KB
DataSetTableTaskService.java 17KB
PermissionService.java 15KB
MapService.java 15KB
ExcelXlsReader.java 15KB
XssAndSqlHttpServletRequestWrapper.java 15KB
EsProvider.java 14KB
AuthUserServiceImpl.java 14KB
ScheduleManager.java 14KB
DataSetTableController.java 13KB
PluginService.java 13KB
SysUserController.java 13KB
ViewPluginBaseServiceImpl.java 13KB
DataSetTableFieldController.java 13KB
DirectFieldService.java 12KB
PanelLinkService.java 12KB
SysMsgService.java 12KB
EmailService.java 12KB
SysStartupJobExample.java 12KB
DataSetTableTaskLogService.java 11KB
XLarkServer.java 10KB
EngineService.java 10KB
XDingtalkServer.java 10KB
共 2000 条
- 1
- 2
- 3
- 4
- 5
- 6
- 20
资源评论
老王学长
- 粉丝: 845
- 资源: 59
上传资源 快速赚钱
- 我的内容管理 展开
- 我的资源 快来上传第一个资源
- 我的收益 登录查看自己的收益
- 我的积分 登录查看自己的积分
- 我的C币 登录后查看C币余额
- 我的收藏
- 我的下载
- 下载帮助
安全验证
文档复制为VIP权益,开通VIP直接复制
信息提交成功