人人可用的开源数据可视化分析工具

package io.dataease.service.dataset; import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.google.gson.reflect.TypeToken; import io.dataease.auth.annotation.DeCleaner; import io.dataease.auth.api.dto.CurrentUserDto; import io.dataease.commons.constants.*; import io.dataease.commons.exception.DEException; import io.dataease.commons.utils.*; import io.dataease.controller.ResultHolder; import io.dataease.controller.request.dataset.DataSetExportRequest; import io.dataease.controller.request.dataset.DataSetGroupRequest; import io.dataease.controller.request.dataset.DataSetTableRequest; import io.dataease.controller.request.dataset.DataSetTaskRequest; import io.dataease.controller.response.DataSetDetail; import io.dataease.controller.sys.base.BaseGridRequest; import io.dataease.controller.sys.base.ConditionEntity; import io.dataease.dto.SysLogDTO; import io.dataease.dto.dataset.*; import io.dataease.dto.dataset.union.UnionDTO; import io.dataease.dto.dataset.union.UnionItemDTO; import io.dataease.dto.dataset.union.UnionParamDTO; import io.dataease.exception.DataEaseException; import io.dataease.ext.ExtDataSetGroupMapper; import io.dataease.ext.ExtDataSetTableMapper; import io.dataease.ext.UtilMapper; import io.dataease.i18n.Translator; import io.dataease.listener.util.CacheUtils; import io.dataease.plugins.common.base.domain.*; import io.dataease.plugins.common.base.mapper.*; import io.dataease.plugins.common.constants.DatasetType; import io.dataease.plugins.common.constants.DatasourceTypes; import io.dataease.plugins.common.constants.DeTypeConstants; import io.dataease.plugins.common.dto.chart.ChartViewFieldDTO; import io.dataease.plugins.common.dto.dataset.SqlVariableDetails; import io.dataease.plugins.common.dto.datasource.DataSourceType; import io.dataease.plugins.common.dto.datasource.TableField; import io.dataease.plugins.common.request.datasource.DatasourceRequest; import io.dataease.plugins.common.request.permission.DataSetRowPermissionsTreeDTO; import io.dataease.plugins.common.request.permission.DatasetRowPermissionsTreeObj; import io.dataease.plugins.datasource.provider.Provider; import io.dataease.plugins.datasource.query.QueryProvider; import io.dataease.plugins.loader.ClassloaderResponsity; import io.dataease.plugins.xpack.auth.dto.request.ColumnPermissionItem; import io.dataease.provider.DDLProvider; import io.dataease.provider.ProviderFactory; import io.dataease.provider.datasource.JdbcProvider; import io.dataease.service.chart.util.ChartDataBuild; import io.dataease.service.datasource.DatasourceService; import io.dataease.service.engine.EngineService; import io.dataease.service.sys.SysAuthService; import net.sf.jsqlparser.expression.Alias; import net.sf.jsqlparser.expression.BinaryExpression; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.Parenthesis; import net.sf.jsqlparser.expression.operators.conditional.AndExpression; import net.sf.jsqlparser.expression.operators.conditional.OrExpression; import net.sf.jsqlparser.expression.operators.relational.*; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.statement.Statement; import net.sf.jsqlparser.statement.select.*; import net.sf.jsqlparser.util.deparser.ExpressionDeParser; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.poi.hssf.usermodel.HSSFDateUtil; import org.apache.poi.ss.usermodel.*; import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; import javax.annotation.Resource; import javax.servlet.http.HttpServletResponse; import java.io.*; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; import java.text.MessageFormat; import java.text.NumberFormat; import java.text.SimpleDateFormat; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; /** * @Author gin * @Date 2021/2/23 2:54 下午 */ @Service @Transactional(rollbackFor = Exception.class) public class DataSetTableService { @Resource private DatasetTableMapper datasetTableMapper; @Resource private DatasourceMapper datasourceMapper; @Resource private DataSetTableFieldsService dataSetTableFieldsService; @Resource private DataSetTableTaskService dataSetTableTaskService; @Resource private CommonThreadPool commonThreadPool; @Resource private ExtractDataService extractDataService; @Resource private ExtDataSetTableMapper extDataSetTableMapper; @Resource private DatasetTableIncrementalConfigMapper datasetTableIncrementalConfigMapper; @Resource private DataSetTableUnionService dataSetTableUnionService; @Resource private QrtzSchedulerStateMapper qrtzSchedulerStateMapper; @Resource private DatasetTableTaskLogMapper datasetTableTaskLogMapper; @Resource private ExtDataSetGroupMapper extDataSetGroupMapper; @Resource private DatasetTableFieldMapper datasetTableFieldMapper; @Resource private PermissionService permissionService; @Resource private EngineService engineService; @Resource private SysAuthService sysAuthService; @Resource private ChartViewMapper chartViewMapper; @Resource private DataSetTableTaskLogService dataSetTableTaskLogService; @Resource private PermissionsTreeService permissionsTreeService; @Resource private DatasourceService datasourceService; @Resource private DatasetSqlLogMapper datasetSqlLogMapper; private static boolean isUpdatingDatasetTableStatus = false; private static final String lastUpdateTime = "${__last_update_time__}"; private static final String currentUpdateTime = "${__current_update_time__}"; public static final String regex = "\\$\\{(.*?)\\}"; private static final String SubstitutedParams = "DATAEASE_PATAMS_BI"; private static final String SubstitutedSql = " 'BI' = 'BI' "; private static final String SubstitutedSqlVirtualData = " 1 > 2 "; @Value("${upload.file.path}") private String path; private static final Logger logger = LoggerFactory.getLogger(ClassloaderResponsity.class); @DeCleaner(value = DePermissionType.DATASET, key = "sceneId") public List<DatasetTable> batchInsert(List<DataSetTableRequest> datasetTable) throws Exception { // 保存之前校验table名称 checkNames(datasetTable); List<DatasetTable> list = new ArrayList<>(); for (DataSetTableRequest table : datasetTable) { list.add(save(table)); // 清理权限缓存 CacheUtils.removeAll(AuthConstants.USER_DATASET_NAME); CacheUtils.removeAll(AuthConstants.ROLE_DATASET_NAME); CacheUtils.removeAll(AuthConstants.DEPT_DATASET_NAME); } return list; } private void extractData(DataSetTableRequest datasetTable) throws Exception { if (datasetTable.getMode() == 1 && StringUtils.isNotEmpty(datasetTable.getSyncType()) && datasetTable.getSyncType().equalsIgnoreCase("sync_now")) { DataSetTaskRequest dataSetTaskRequest = new DataSetTaskRequest(); DatasetTableTask datasetTableTask = new DatasetTableTask(); datasetTableTask.setTableId(datasetTable.getId()); datasetTableTask.setRate(ScheduleType.SIMPLE.toString()); datasetTableTask.setType("all_scope"); datasetTableTask.setName(datasetTable.getName() + " 更新设置-" + System.currentTimeMillis());