package io.dataease.service.dataset;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelReader;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import com.alibaba.excel.metadata.CellData;
import com.alibaba.excel.read.metadata.ReadSheet;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.TypeReference;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.reflect.TypeToken;
import io.dataease.auth.annotation.DeCleaner;
import io.dataease.auth.api.dto.CurrentUserDto;
import io.dataease.commons.constants.*;
import io.dataease.commons.utils.*;
import io.dataease.controller.ResultHolder;
import io.dataease.controller.dataset.request.DataSetTaskInstanceGridRequest;
import io.dataease.controller.request.dataset.DataSetExportRequest;
import io.dataease.controller.request.dataset.DataSetGroupRequest;
import io.dataease.controller.request.dataset.DataSetTableRequest;
import io.dataease.controller.request.dataset.DataSetTaskRequest;
import io.dataease.controller.response.DataSetDetail;
import io.dataease.dto.SysLogDTO;
import io.dataease.dto.dataset.*;
import io.dataease.plugins.common.dto.dataset.DataTableInfoCustomUnion;
import io.dataease.plugins.common.dto.dataset.DataTableInfoDTO;
import io.dataease.plugins.common.dto.dataset.ExcelSheetData;
import io.dataease.plugins.common.dto.dataset.union.UnionDTO;
import io.dataease.plugins.common.dto.dataset.union.UnionItemDTO;
import io.dataease.plugins.common.dto.dataset.union.UnionParamDTO;
import io.dataease.ext.ExtDataSetGroupMapper;
import io.dataease.ext.ExtDataSetTableMapper;
import io.dataease.ext.UtilMapper;
import io.dataease.i18n.Translator;
import io.dataease.listener.util.CacheUtils;
import io.dataease.plugins.common.base.domain.*;
import io.dataease.plugins.common.base.mapper.*;
import io.dataease.plugins.common.constants.DatasetType;
import io.dataease.plugins.common.constants.DatasourceTypes;
import io.dataease.plugins.common.constants.DeTypeConstants;
import io.dataease.plugins.common.dto.dataset.SqlVariableDetails;
import io.dataease.plugins.common.dto.datasource.DataSourceType;
import io.dataease.plugins.common.dto.datasource.TableField;
import io.dataease.plugins.common.exception.DataEaseException;
import io.dataease.plugins.common.request.datasource.DatasourceRequest;
import io.dataease.plugins.common.request.permission.DataSetRowPermissionsTreeDTO;
import io.dataease.plugins.common.request.permission.DatasetRowPermissionsTreeObj;
import io.dataease.plugins.common.util.ClassloaderResponsity;
import io.dataease.plugins.datasource.provider.Provider;
import io.dataease.plugins.datasource.query.QueryProvider;
import io.dataease.plugins.xpack.auth.dto.request.ColumnPermissionItem;
import io.dataease.plugins.datasource.provider.DDLProvider;
import io.dataease.plugins.datasource.provider.ProviderFactory;
import io.dataease.provider.datasource.JdbcProvider;
import io.dataease.service.chart.util.ChartDataBuild;
import io.dataease.service.datasource.DatasourceService;
import io.dataease.service.engine.EngineService;
import io.dataease.service.sys.SysAuthService;
import lombok.Data;
import net.sf.jsqlparser.expression.Alias;
import net.sf.jsqlparser.expression.BinaryExpression;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.Parenthesis;
import net.sf.jsqlparser.expression.operators.conditional.AndExpression;
import net.sf.jsqlparser.expression.operators.conditional.OrExpression;
import net.sf.jsqlparser.expression.operators.relational.*;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.statement.Statement;
import net.sf.jsqlparser.statement.select.*;
import net.sf.jsqlparser.util.deparser.ExpressionDeParser;
import net.sf.jsqlparser.util.deparser.SelectDeParser;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.hssf.usermodel.HSSFDateUtil;
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.text.MessageFormat;
import java.text.NumberFormat;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* @Author gin
* @Date 2021/2/23 2:54 下午
*/
@Service
@Transactional(rollbackFor = Exception.class)
public class DataSetTableService {
@Resource
private DatasetTableMapper datasetTableMapper;
@Resource
private DatasourceMapper datasourceMapper;
@Resource
private DataSetTableFieldsService dataSetTableFieldsService;
@Resource
private DataSetTableTaskService dataSetTableTaskService;
@Resource
private CommonThreadPool commonThreadPool;
@Resource
private ExtractDataService extractDataService;
@Resource
private ExtDataSetTableMapper extDataSetTableMapper;
@Resource
private DatasetTableIncrementalConfigMapper datasetTableIncrementalConfigMapper;
@Resource
private DataSetTableUnionService dataSetTableUnionService;
@Resource
private QrtzSchedulerStateMapper qrtzSchedulerStateMapper;
@Resource
private DatasetTableTaskLogMapper datasetTableTaskLogMapper;
@Resource
private ExtDataSetGroupMapper extDataSetGroupMapper;
@Resource
private DatasetTableFieldMapper datasetTableFieldMapper;
@Resource
private PermissionService permissionService;
@Resource
private EngineService engineService;
@Resource
private SysAuthService sysAuthService;
@Resource
private ChartViewMapper chartViewMapper;
@Resource
private DataSetTableTaskLogService dataSetTableTaskLogService;
@Resource
private PermissionsTreeService permissionsTreeService;
@Resource
private DatasourceService datasourceService;
@Resource
private DatasetSqlLogMapper datasetSqlLogMapper;
private static boolean isUpdatingDatasetTableStatus = false;
private static final String lastUpdateTime = "${__last_update_time__}";
private static final String currentUpdateTime = "${__current_update_time__}";
public static final String regex = "\\$\\{(.*?)\\}";
private static final String SubstitutedParams = "DATAEASE_PATAMS_BI";
private static final String SubstitutedSql = " 'BI' = 'BI' ";
private static final String SubstitutedSqlVirtualData = " 1 > 2 ";
@Value("${upload.file.path}")
private String path;
@Value("${export.dataset.limit:100000}")
private int limit;
private static final Logger logger = LoggerFactory.getLogger(ClassloaderResponsity.class);
@DeCleaner(value = DePermissionType.DATASET, key = "sceneId")
public List<DatasetTable> batchInsert(List<DataSetTableRequest> datasetTable) throws Exception {
// 保存之前校验table名称
checkNames(datasetTable);
List<DatasetTable> list = new ArrayList<>();
for (DataSetTableRequest table : datasetTable) {
list.add(save(table));
// 清理权限缓存
CacheUtils.removeAll(AuthConstants.USER_DATASET_NAME);
CacheUtils.removeAll(AuthConstants.ROLE_DATASET_NAME);
CacheUtils.removeAll(AuthConstants.DEPT_DATASET_NAME);
}
return list;
}
public void saveKey(DatasetTable datasetTable, Datas