数据预览优化，SQL组装移到后端

2021-05-20 00:04:07 +08:00
parent dcf487d55c
commit 8fa04b9973
12 changed files with 328 additions and 100 deletions
--- a/zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/controller/DbDataViewController.java
+++ b/zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/controller/DbDataViewController.java
@@ -0,0 +1,106 @@
+package com.zyplayer.doc.db.controller;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.fastjson.util.TypeUtils;
+import com.zyplayer.doc.core.annotation.AuthMan;
+import com.zyplayer.doc.core.json.ResponseJson;
+import com.zyplayer.doc.core.util.StringUtil;
+import com.zyplayer.doc.data.config.security.DocUserUtil;
+import com.zyplayer.doc.data.repository.support.consts.DocAuthConst;
+import com.zyplayer.doc.db.controller.param.DataViewParam;
+import com.zyplayer.doc.db.framework.consts.DbAuthType;
+import com.zyplayer.doc.db.framework.db.mapper.base.ExecuteParam;
+import com.zyplayer.doc.db.framework.db.mapper.base.ExecuteResult;
+import com.zyplayer.doc.db.framework.db.mapper.base.ExecuteType;
+import com.zyplayer.doc.db.framework.db.mapper.base.SqlExecutor;
+import com.zyplayer.doc.db.framework.json.DocDbResponseJson;
+import com.zyplayer.doc.db.framework.utils.JSONUtil;
+import com.zyplayer.doc.db.service.DbBaseFactory;
+import com.zyplayer.doc.db.service.DbBaseService;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import javax.annotation.Resource;
+import java.util.*;
+
+/**
+ * 表数据查看控制器
+ *
+ * @author 暮光：城中城
+ * @since 2021年5月16日
+ */
+@AuthMan
+@RestController
+@RequestMapping("/zyplayer-doc-db/data-view")
+public class DbDataViewController {
+	private static Logger logger = LoggerFactory.getLogger(DbDataViewController.class);
+	
+	@Resource
+	SqlExecutor sqlExecutor;
+	@Resource
+	DbBaseFactory dbBaseFactory;
+	
+	@PostMapping(value = "/query")
+	public ResponseJson execute(DataViewParam dataViewParam) {
+		boolean manageAuth = DocUserUtil.haveAuth(DocAuthConst.DB_DATASOURCE_MANAGE);
+		boolean select = DocUserUtil.haveCustomAuth(DbAuthType.SELECT.getName(), DocAuthConst.DB + dataViewParam.getSourceId());
+		boolean update = DocUserUtil.haveCustomAuth(DbAuthType.UPDATE.getName(), DocAuthConst.DB + dataViewParam.getSourceId());
+		if (!manageAuth && !select && !update) {
+			return DocDbResponseJson.warn("没有该数据源的执行权限");
+		}
+		// 数据查询
+		ExecuteType executeType = (manageAuth || update) ? ExecuteType.ALL : ExecuteType.SELECT;
+		DbBaseService dbBaseService = dbBaseFactory.getDbBaseService(dataViewParam.getSourceId());
+		String queryPageSql = dbBaseService.getQueryPageSql(dataViewParam);
+		ExecuteResult executeResult = this.query(dataViewParam.getSourceId(), dataViewParam.getExecuteId(), executeType, queryPageSql);
+		// 数据组装
+		List<String> resultList = new LinkedList<>();
+		resultList.add(JSON.toJSONString(executeResult, JSONUtil.serializeConfig, SerializerFeature.WriteMapNullValue));
+		DocDbResponseJson responseJson = DocDbResponseJson.ok(resultList);
+		// 计算总条数，第一页才查询总条数
+		if (CollectionUtils.isNotEmpty(executeResult.getResult()) && Objects.equals(dataViewParam.getPageNum(), 1)) {
+			responseJson.setTotal((long) executeResult.getResult().size());
+			if (executeResult.getResult().size() >= dataViewParam.getPageSize()) {
+				String queryCountSql = dbBaseService.getQueryCountSql(dataViewParam);
+				ExecuteResult countResult = this.query(dataViewParam.getSourceId(), dataViewParam.getExecuteId(), executeType, queryCountSql);
+				if (CollectionUtils.isNotEmpty(countResult.getResult()) && MapUtils.isNotEmpty(countResult.getResult().get(0))) {
+					Map<String, Object> countMap = countResult.getResult().get(0);
+					Optional<Object> countAny = countMap.values().stream().findAny();
+					countAny.ifPresent(count -> responseJson.setTotal(TypeUtils.castToLong(count)));
+				}
+			}
+		}
+		return responseJson;
+	}
+	
+	/**
+	 * 执行数据查询
+	 *
+	 * @param sourceId
+	 * @param executeId
+	 * @param executeType
+	 * @param executeSql
+	 * @return
+	 */
+	private ExecuteResult query(Long sourceId, String executeId, ExecuteType executeType, String executeSql) {
+		try {
+			ExecuteParam executeParam = new ExecuteParam();
+			executeParam.setDatasourceId(sourceId);
+			executeParam.setExecuteId(executeId);
+			executeParam.setExecuteType(executeType);
+			executeParam.setSql(executeSql);
+			executeParam.setMaxRows(1000);
+			return sqlExecutor.execute(executeParam);
+		} catch (Exception e) {
+			logger.error("执行出错", e);
+			return ExecuteResult.error(StringUtil.getException(e), executeSql);
+		}
+	}
+}
+
--- a/zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/controller/DbSqlExecutorController.java
+++ b/zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/controller/DbSqlExecutorController.java
@@ -1,9 +1,7 @@
 package com.zyplayer.doc.db.controller;

 import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.serializer.SerializeConfig;
 import com.alibaba.fastjson.serializer.SerializerFeature;
-import com.alibaba.fastjson.serializer.SimpleDateFormatSerializer;
 import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
 import com.zyplayer.doc.core.annotation.AuthMan;
 import com.zyplayer.doc.core.json.ResponseJson;
@@ -21,6 +19,7 @@ import com.zyplayer.doc.db.framework.db.mapper.base.ExecuteResult;
 import com.zyplayer.doc.db.framework.db.mapper.base.ExecuteType;
 import com.zyplayer.doc.db.framework.db.mapper.base.SqlExecutor;
 import com.zyplayer.doc.db.framework.json.DocDbResponseJson;
+import com.zyplayer.doc.db.framework.utils.JSONUtil;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -29,7 +28,6 @@ import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;

 import javax.annotation.Resource;
-import java.sql.Timestamp;
 import java.util.Date;
 import java.util.LinkedList;
 import java.util.List;
@@ -84,10 +82,7 @@ public class DbSqlExecutorController {
 				executeParam.setSql(sqlItem);
 				executeParam.setMaxRows(1000);
 				ExecuteResult executeResult = sqlExecutor.execute(executeParam);
-				SerializeConfig mapping = new SerializeConfig();
-				mapping.put(Date.class, new SimpleDateFormatSerializer("yyyy-MM-dd HH:mm:ss"));
-				mapping.put(Timestamp.class, new SimpleDateFormatSerializer("yyyy-MM-dd HH:mm:ss"));
-				String resultJsonStr = JSON.toJSONString(executeResult, mapping, SerializerFeature.WriteMapNullValue);
+				String resultJsonStr = JSON.toJSONString(executeResult, JSONUtil.serializeConfig, SerializerFeature.WriteMapNullValue);
 				resultList.add(resultJsonStr);
 			} catch (Exception e) {
 				logger.error("执行出错", e);
@@ -146,5 +141,6 @@ public class DbSqlExecutorController {
 		}
 		return DocDbResponseJson.ok();
 	}
+	
 }

--- a/zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/controller/param/DataViewParam.java
+++ b/zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/controller/param/DataViewParam.java
@@ -0,0 +1,89 @@
+package com.zyplayer.doc.db.controller.param;
+
+public class DataViewParam {
+	private Long sourceId;
+	private String executeId;
+	private Integer pageSize;
+	private Integer pageNum;
+	private String dbName;
+	private String tableName;
+	private String orderColumn;
+	private String orderType;
+	private String condition;
+	
+	public Integer getOffset() {
+		return ((this.pageNum - 1) * this.pageSize);
+	}
+	
+	public Long getSourceId() {
+		return sourceId;
+	}
+	
+	public void setSourceId(Long sourceId) {
+		this.sourceId = sourceId;
+	}
+	
+	public String getExecuteId() {
+		return executeId;
+	}
+	
+	public void setExecuteId(String executeId) {
+		this.executeId = executeId;
+	}
+	
+	public Integer getPageSize() {
+		return pageSize;
+	}
+	
+	public void setPageSize(Integer pageSize) {
+		this.pageSize = pageSize;
+	}
+	
+	public Integer getPageNum() {
+		return pageNum;
+	}
+	
+	public void setPageNum(Integer pageNum) {
+		this.pageNum = pageNum;
+	}
+	
+	public String getDbName() {
+		return dbName;
+	}
+	
+	public void setDbName(String dbName) {
+		this.dbName = dbName;
+	}
+	
+	public String getTableName() {
+		return tableName;
+	}
+	
+	public void setTableName(String tableName) {
+		this.tableName = tableName;
+	}
+	
+	public String getOrderColumn() {
+		return orderColumn;
+	}
+	
+	public void setOrderColumn(String orderColumn) {
+		this.orderColumn = orderColumn;
+	}
+	
+	public String getOrderType() {
+		return orderType;
+	}
+	
+	public void setOrderType(String orderType) {
+		this.orderType = orderType;
+	}
+	
+	public String getCondition() {
+		return condition;
+	}
+	
+	public void setCondition(String condition) {
+		this.condition = condition;
+	}
+}
--- a/zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/framework/utils/JSONUtil.java
+++ b/zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/framework/utils/JSONUtil.java
@@ -0,0 +1,17 @@
+package com.zyplayer.doc.db.framework.utils;
+
+import com.alibaba.fastjson.serializer.SerializeConfig;
+import com.alibaba.fastjson.serializer.SimpleDateFormatSerializer;
+
+import java.sql.Timestamp;
+import java.util.Date;
+
+public class JSONUtil {
+	
+	public static SerializeConfig serializeConfig = new SerializeConfig();
+	
+	static {
+		serializeConfig.put(Date.class, new SimpleDateFormatSerializer("yyyy-MM-dd HH:mm:ss"));
+		serializeConfig.put(Timestamp.class, new SimpleDateFormatSerializer("yyyy-MM-dd HH:mm:ss"));
+	}
+}
--- a/zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/service/DbBaseService.java
+++ b/zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/service/DbBaseService.java
@@ -3,6 +3,7 @@ package com.zyplayer.doc.db.service;
 import com.zyplayer.doc.core.exception.ConfirmException;
 import com.zyplayer.doc.data.config.security.DocUserUtil;
 import com.zyplayer.doc.data.repository.support.consts.DocAuthConst;
+import com.zyplayer.doc.db.controller.param.DataViewParam;
 import com.zyplayer.doc.db.controller.param.ProcedureListParam;
 import com.zyplayer.doc.db.controller.vo.TableColumnVo;
 import com.zyplayer.doc.db.controller.vo.TableDdlVo;
@@ -335,4 +336,28 @@ public abstract class DbBaseService {
 		// 需要各数据服务自己实现，各数据库产品的实现都不一样
 		throw new ConfirmException("暂未支持的数据库类型");
 	}
+	
+	/**
+	 * 获取分页查询的SQL
+	 *
+	 * @return 分页查询的SQL
+	 * @author 暮光：城中城
+	 * @since 2020年4月24日
+	 */
+	public String getQueryPageSql(DataViewParam dataViewParam) {
+		// 需要各数据服务自己实现，各数据库产品的实现都不一样
+		throw new ConfirmException("暂未支持的数据库类型");
+	}
+	
+	/**
+	 * 获取查询总条数的SQL
+	 *
+	 * @return 查询总条数的SQL
+	 * @author 暮光：城中城
+	 * @since 2020年4月24日
+	 */
+	public String getQueryCountSql(DataViewParam dataViewParam) {
+		// 需要各数据服务自己实现，各数据库产品的实现都不一样
+		throw new ConfirmException("暂未支持的数据库类型");
+	}
 }
--- a/zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/service/HiveServiceImpl.java
+++ b/zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/service/HiveServiceImpl.java
@@ -1,5 +1,10 @@
 package com.zyplayer.doc.db.service;

+import com.zyplayer.doc.core.exception.ConfirmException;
+import com.zyplayer.doc.db.controller.vo.TableStatusVo;
+import com.zyplayer.doc.db.framework.db.dto.QueryTableColumnDescDto;
+import com.zyplayer.doc.db.framework.db.dto.TableColumnDescDto;
+import com.zyplayer.doc.db.framework.db.dto.TableDescDto;
 import com.zyplayer.doc.db.framework.db.dto.TableInfoDto;
 import com.zyplayer.doc.db.framework.db.enums.DatabaseProductEnum;
 import com.zyplayer.doc.db.framework.db.mapper.base.BaseMapper;
@@ -24,4 +29,29 @@ public class HiveServiceImpl extends DbBaseService {
 		}
 		return tableList;
 	}
+	
+	@Override
+	public List<TableColumnDescDto> getTableColumnDescList(Long sourceId, String tableName) {
+		throw new ConfirmException("不支持的操作");
+	}
+	
+	@Override
+	public List<QueryTableColumnDescDto> getTableAndColumnBySearch(Long sourceId, String dbName, String searchText) {
+		throw new ConfirmException("不支持的操作");
+	}
+	
+	@Override
+	public List<TableDescDto> getTableDescList(Long sourceId, String dbName, String tableName) {
+		throw new ConfirmException("不支持的操作");
+	}
+	
+	@Override
+	public void updateTableDesc(Long sourceId, String dbName, String tableName, String newDesc) {
+		throw new ConfirmException("不支持的操作");
+	}
+	
+	@Override
+	public void updateTableColumnDesc(Long sourceId, String dbName, String tableName, String columnName, String newDesc) {
+		throw new ConfirmException("不支持的操作");
+	}
 }
--- a/zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/service/MysqlServiceImpl.java
+++ b/zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/service/MysqlServiceImpl.java
@@ -1,6 +1,7 @@
 package com.zyplayer.doc.db.service;

 import cn.hutool.core.util.RandomUtil;
+import com.zyplayer.doc.db.controller.param.DataViewParam;
 import com.zyplayer.doc.db.controller.vo.TableDdlVo;
 import com.zyplayer.doc.db.framework.db.dto.ColumnInfoDto;
 import com.zyplayer.doc.db.framework.db.dto.ProcedureDto;
@@ -125,4 +126,28 @@ public class MysqlServiceImpl extends DbBaseService {
 			return ExecuteResult.error(e.getMessage(), procSql);
 		}
 	}
+	
+	@Override
+	public String getQueryPageSql(DataViewParam dataViewParam) {
+		StringBuilder sqlSb = new StringBuilder();
+		sqlSb.append(String.format("select * from %s.%s", dataViewParam.getDbName(), dataViewParam.getTableName()));
+		if (StringUtils.isNotBlank(dataViewParam.getCondition())) {
+			sqlSb.append(String.format(" where %s", dataViewParam.getCondition()));
+		}
+		if (StringUtils.isNotBlank(dataViewParam.getOrderColumn()) && StringUtils.isNotBlank(dataViewParam.getOrderType())) {
+			sqlSb.append(String.format(" order by %s %s", dataViewParam.getOrderColumn(), dataViewParam.getOrderType()));
+		}
+		sqlSb.append(String.format(" limit %s offset %s", dataViewParam.getPageSize(), dataViewParam.getOffset()));
+		return sqlSb.toString();
+	}
+	
+	@Override
+	public String getQueryCountSql(DataViewParam dataViewParam) {
+		StringBuilder sqlSb = new StringBuilder();
+		sqlSb.append(String.format("select count(1) as counts from %s.%s", dataViewParam.getDbName(), dataViewParam.getTableName()));
+		if (StringUtils.isNotBlank(dataViewParam.getCondition())) {
+			sqlSb.append(String.format(" where %s", dataViewParam.getCondition()));
+		}
+		return sqlSb.toString();
+	}
 }