sql执行器数据量过大自动开启分页

已发现bug修复
This commit is contained in:
diantu
2023-02-22 19:16:31 +08:00
parent d8ea5c34ec
commit fdd9683a7f
18 changed files with 328 additions and 199 deletions

View File

@@ -2,6 +2,9 @@ package com.zyplayer.doc.db.controller;
import com.alibaba.druid.DbType;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.ast.statement.SQLSelectQuery;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.parser.SQLStatementParser;
import com.alibaba.fastjson.JSON;
import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
@@ -15,6 +18,7 @@ import com.zyplayer.doc.data.repository.support.consts.DocSysModuleType;
import com.zyplayer.doc.data.repository.support.consts.DocSysType;
import com.zyplayer.doc.data.service.manage.DbFavoriteService;
import com.zyplayer.doc.data.service.manage.DbHistoryService;
import com.zyplayer.doc.db.controller.param.DataViewParam;
import com.zyplayer.doc.db.framework.consts.DbAuthType;
import com.zyplayer.doc.db.framework.db.mapper.base.*;
import com.zyplayer.doc.db.framework.db.transfer.SqlParseUtil;
@@ -54,7 +58,7 @@ public class DbSqlExecutorController {
DatabaseServiceFactory databaseServiceFactory;
@PostMapping(value = "/execute")
public DocDbResponseJson execute(Long sourceId, String executeId, String dbName, String sql, String params) {
public DocDbResponseJson execute(Long sourceId, String executeId, String dbName, String sql, String params,Integer pageSize,Integer pageNum) {
if (StringUtils.isBlank(sql)) {
return DocDbResponseJson.warn("执行的SQL不能为空");
}
@@ -71,7 +75,7 @@ public class DbSqlExecutorController {
// 参数处理
Map<String, Object> paramMap = JSON.parseObject(params);
// 解析出多个执行的SQL
List<String> analysisQuerySqlList = new LinkedList<>();
List<Map<String,Object>> analysisQuerySqlList = new LinkedList<Map<String,Object>>();
try {
String driverClassName = dbBaseService.getDatabaseProduct().getDriverClassName();
List<SQLStatement> sqlStatements = new ArrayList<SQLStatement>();
@@ -79,7 +83,23 @@ public class DbSqlExecutorController {
DbType dbType = SQLTransformUtils.getDbTypeByDriverClassName(driverClassName);
sqlStatements = new SQLStatementParser(sql,dbType).parseStatementList();
for (SQLStatement sqlStatement : sqlStatements) {
analysisQuerySqlList.add(sqlStatement.toString());
StringBuffer sb = new StringBuffer(sqlStatement.toString());
if(sb.length()>0&&';' == (sb.charAt(sb.length()-1))){
sb.deleteCharAt(sb.length()-1);
}
Map<String,Object> map = new HashMap<String,Object>();
//原始sql
map.put("originalSql",sb);
//sql解析类型
map.put("sqlType","");
//获取数据总量sql
map.put("getAllCountSql","");
//判断sql解析类型
if(sqlStatement instanceof SQLSelectStatement){
map.put("getAllCountSql","select count(1) from ("+sb+") r");
map.put("sqlType","select");
}
analysisQuerySqlList.add(map);
}
} catch (Exception e) {
return DocDbResponseJson.warn("SQL解析失败" + e.getMessage());
@@ -89,20 +109,42 @@ public class DbSqlExecutorController {
return DocDbResponseJson.warn("单次执行最多支持20条语句同时执行当前语句条数" + analysisQuerySqlList.size());
}
List<ColumnExecuteResult> resultList = new LinkedList<>();
for (String singleSql : analysisQuerySqlList) {
for (Map<String, Object> map : analysisQuerySqlList) {
ColumnExecuteResult executeResult;
ColumnExecuteResult executeCountResult;
//原始sql
String originalSql = map.get("originalSql").toString();
try {
ExecuteType executeType = (manageAuth || update) ? ExecuteType.ALL : ExecuteType.SELECT;
ExecuteParam executeParam = SqlParseUtil.getSingleExecuteParam(singleSql, paramMap);
ExecuteParam executeParam = new ExecuteParam();
executeParam.setDatasourceId(sourceId);
executeParam.setExecuteId(executeId);
executeParam.setExecuteType(executeType);
executeParam.setPrefixSql(useDbSql);
executeParam.setMaxRows(1000);
//sql解析类型为select
if(map.get("sqlType").equals("select")){
//获取总数据量sql
String getAllCountSql = map.get("getAllCountSql").toString();
executeParam = SqlParseUtil.getSingleExecuteParam(executeParam,getAllCountSql, paramMap);
executeCountResult = columnSqlExecutor.execute(executeParam);
List<List<Object>> data = executeCountResult.getData();
long count = Long.parseLong(data.get(0).get(0)+"");
//总数据量大于1000进行分页
if(count>1000){
String pageSql = dbBaseService.getQueryPageSqlBySql(originalSql,pageSize,pageNum);
executeParam = SqlParseUtil.getSingleExecuteParam(executeParam,pageSql, paramMap);
executeResult = columnSqlExecutor.execute(executeParam);
executeResult.setSelectCount(count);
resultList.add(executeResult);
continue;
}
}
executeParam = SqlParseUtil.getSingleExecuteParam(executeParam,originalSql, paramMap);
executeResult = columnSqlExecutor.execute(executeParam);
} catch (Exception e) {
logger.error("执行出错", e);
executeResult = ColumnExecuteResult.error(singleSql, e.getMessage(), e);
executeResult = ColumnExecuteResult.error(originalSql, e.getMessage(), e);
}
resultList.add(executeResult);
}

View File

@@ -19,22 +19,23 @@ public class ColumnExecuteResult {
private Exception exception;
private String executeSql;
private Integer updateCount;
private long selectCount;
// 查询结果表头
private List<String> header;
// 查询结果数据
private List<List<Object>> data;
public ColumnExecuteResult() {
this.updateCount = 0;
}
public static ColumnExecuteResult ok(String sql) {
ColumnExecuteResult result = new ColumnExecuteResult();
result.setExecuteSql(sql);
result.setErrCode(ExecuteResultCode.SUCCESS);
return result;
}
public static ColumnExecuteResult warn(String sql, String errMsg) {
ColumnExecuteResult result = new ColumnExecuteResult();
result.setExecuteSql(sql);
@@ -42,7 +43,7 @@ public class ColumnExecuteResult {
result.setErrCode(ExecuteResultCode.WARN);
return result;
}
public static ColumnExecuteResult error(String sql, String errMsg, Exception e) {
ColumnExecuteResult result = new ColumnExecuteResult();
result.setExecuteSql(sql);
@@ -51,13 +52,13 @@ public class ColumnExecuteResult {
result.setErrCode(ExecuteResultCode.ERROR);
return result;
}
public static class ExecuteResultCode {
public static Integer SUCCESS = 0;
public static Integer WARN = -1;
public static Integer ERROR = -2;
}
/**
* 判断错误并抛出异常
*/

View File

@@ -28,13 +28,13 @@ import java.util.concurrent.ConcurrentHashMap;
@Repository
public class ColumnSqlExecutor {
private static Logger logger = LoggerFactory.getLogger(SqlExecutor.class);
@Resource
DatabaseRegistrationBean databaseRegistrationBean;
// 执行中的PreparedStatement信息用于强制取消执行
private static final Map<String, PreparedStatement> statementMap = new ConcurrentHashMap<>();
/**
* 取消执行
*
@@ -54,7 +54,7 @@ public class ColumnSqlExecutor {
}
return false;
}
/**
* 执行sql返回结果
*
@@ -65,7 +65,7 @@ public class ColumnSqlExecutor {
DatabaseFactoryBean factoryBean = databaseRegistrationBean.getOrCreateFactoryById(param.getDatasourceId());
return this.execute(factoryBean, param, null);
}
/**
* 执行sql返回结果
*
@@ -76,7 +76,7 @@ public class ColumnSqlExecutor {
DatabaseFactoryBean factoryBean = databaseRegistrationBean.getOrCreateFactoryById(param.getDatasourceId());
return this.execute(factoryBean, param, handler);
}
/**
* 执行sql可通过handler回调每一行的结果
*

View File

@@ -31,7 +31,7 @@ import java.util.stream.Collectors;
* @since 2019年9月28日
*/
public class SqlParseUtil {
/**
* 通过SQL和参数列表获取处理后的SQL和参数
* @param sql
@@ -58,7 +58,7 @@ public class SqlParseUtil {
}
return executeParamList;
}
/**
* 解析sql如果是insert则把values拼成多个用于支持批量插入
* 不是insert则返回null
@@ -145,7 +145,7 @@ public class SqlParseUtil {
executeParam.setParamList(resultParamList);
return executeParam;
}
/**
* 解析单条操作的SQL
* @param sql
@@ -168,7 +168,29 @@ public class SqlParseUtil {
executeParam.setParamList(resultParamList);
return executeParam;
}
/**
* 解析单条操作的SQL
* @param sql
* @param paramMap
* @return
*/
public static ExecuteParam getSingleExecuteParam(ExecuteParam executeParam,String sql, Map<String, Object> paramMap) {
sql = FillParamUtil.fillSqlParam(sql, paramMap);
SqlSourceBuilder sqlSourceBuilder = new SqlSourceBuilder(new Configuration());
StaticSqlSource parse = (StaticSqlSource) sqlSourceBuilder.parse(sql, Object.class, paramMap);
BoundSql boundSql = parse.getBoundSql(new Object());
List<ParameterMapping> parameterMappings = boundSql.getParameterMappings();
List<Object> resultParamList = new LinkedList<>();
for (ParameterMapping parameterMapping : parameterMappings) {
resultParamList.add(paramMap.get(parameterMapping.getProperty()));
}
executeParam.setSql(boundSql.getSql());
executeParam.setParameterMappings(parameterMappings);
executeParam.setParamList(resultParamList);
return executeParam;
}
/**
* 获取sql语句里面查询的列
*
@@ -190,7 +212,7 @@ public class SqlParseUtil {
}
return null;
}
/**
* 获取sql语句里面查询的列
*
@@ -239,7 +261,7 @@ public class SqlParseUtil {
}
return null;
}
/**
* 获取sql语句里面查询的列
*
@@ -285,7 +307,7 @@ public class SqlParseUtil {
}
return Collections.emptyList();
}
public static void main(String[] args) {
String storageSql = "select * from zyplayer_doc_manage.wiki_space where id=1 group by id";
storageSql = storageSql.replaceAll("(#\\{\\w+})", "'$1'");

View File

@@ -153,17 +153,17 @@ public class SQLTransformUtils {
*/
public static DbType getDbTypeByDriverClassName(String driverClassName) {
DbType dbType;
if(driverClassName.equalsIgnoreCase(DatabaseProductEnum.MYSQL.name())){
if(driverClassName.equalsIgnoreCase(DatabaseProductEnum.MYSQL.getDriverClassName())){
dbType = DbType.mysql;
}else if(driverClassName.equalsIgnoreCase(DatabaseProductEnum.ORACLE.name())){
}else if(driverClassName.equalsIgnoreCase(DatabaseProductEnum.ORACLE.getDriverClassName())){
dbType = DbType.oracle;
}else if(driverClassName.equalsIgnoreCase(DatabaseProductEnum.DM.name())){
}else if(driverClassName.equalsIgnoreCase(DatabaseProductEnum.DM.getDriverClassName())){
dbType = DbType.dm;
}else if(driverClassName.equalsIgnoreCase(DatabaseProductEnum.SQLSERVER.name())){
}else if(driverClassName.equalsIgnoreCase(DatabaseProductEnum.SQLSERVER.getDriverClassName())){
dbType = DbType.sqlserver;
}else if(driverClassName.equalsIgnoreCase(DatabaseProductEnum.POSTGRESQL.name())){
}else if(driverClassName.equalsIgnoreCase(DatabaseProductEnum.POSTGRESQL.getDriverClassName())){
dbType = DbType.postgresql;
}else if(driverClassName.equalsIgnoreCase(DatabaseProductEnum.HIVE.name())){
}else if(driverClassName.equalsIgnoreCase(DatabaseProductEnum.HIVE.getDriverClassName())){
dbType = DbType.hive;
}else{
dbType = DbType.other;

View File

@@ -43,7 +43,7 @@ import java.util.Set;
*/
public abstract class DbBaseService {
private static Logger logger = LoggerFactory.getLogger(DbBaseService.class);
@Resource
SqlExecutor sqlExecutor;
@Resource
@@ -52,7 +52,7 @@ public abstract class DbBaseService {
BaseDownloadService baseDownloadService;
@Resource
DatabaseRegistrationBean databaseRegistrationBean;
/**
* 判断查看权和获取BaseMapper
*
@@ -66,7 +66,7 @@ public abstract class DbBaseService {
}
return baseMapper;
}
/**
* 权限判断
*
@@ -78,14 +78,14 @@ public abstract class DbBaseService {
throw new ConfirmException(noAuthInfo);
}
}
/**
* 获取当前是什么数据源服务
*
* @return 服务类型
*/
public abstract DatabaseProductEnum getDatabaseProduct();
/**
* 获取库列表
*
@@ -97,7 +97,7 @@ public abstract class DbBaseService {
// 需要各数据服务自己实现,各数据库产品的实现都不一样
throw new ConfirmException("暂未支持的数据库类型");
}
/**
* 获取库列表
*
@@ -109,7 +109,7 @@ public abstract class DbBaseService {
BaseMapper baseMapper = this.getViewAuthBaseMapper(sourceId);
return baseMapper.getDatabaseList();
}
/**
* 获取表列表
*
@@ -122,7 +122,7 @@ public abstract class DbBaseService {
BaseMapper baseMapper = this.getViewAuthBaseMapper(sourceId);
return baseMapper.getTableList(dbName);
}
/**
* 获取字段列表
*
@@ -151,7 +151,7 @@ public abstract class DbBaseService {
tableColumnVo.setTableInfo(tableInfoVo);
return tableColumnVo;
}
/**
* 模糊搜索表和字段
*
@@ -166,7 +166,7 @@ public abstract class DbBaseService {
searchText = "%" + searchText + "%";
return baseMapper.getTableAndColumnBySearch(dbName, searchText);
}
/**
* 获取表注释
*
@@ -179,7 +179,7 @@ public abstract class DbBaseService {
BaseMapper baseMapper = this.getViewAuthBaseMapper(sourceId);
return baseMapper.getTableDescList(dbName, tableName);
}
/**
* 增加表注释
*
@@ -192,7 +192,7 @@ public abstract class DbBaseService {
BaseMapper baseMapper = this.getViewAuthBaseMapper(sourceId);
baseMapper.updateTableDesc(dbName, tableName, newDesc);
}
/**
* 增加字段注释
*
@@ -207,7 +207,7 @@ public abstract class DbBaseService {
BaseMapper baseMapper = this.getViewAuthBaseMapper(sourceId);
baseMapper.updateTableColumnDesc(dbName, tableName, columnName, newDesc, null);
}
/**
* 获取表基本信息
*
@@ -223,7 +223,7 @@ public abstract class DbBaseService {
tableStatusVo.setDbType(factoryBean.getDatabaseProduct().name().toLowerCase());
return tableStatusVo;
}
/**
* 获取存储过程列表
*
@@ -235,7 +235,7 @@ public abstract class DbBaseService {
BaseMapper baseMapper = this.getViewAuthBaseMapper(procedureParam.getSourceId());
return baseMapper.getProcedureCount(procedureParam);
}
/**
* 获取存储过程列表
*
@@ -251,7 +251,7 @@ public abstract class DbBaseService {
}
return baseMapper.getProcedureList(procedureParam);
}
/**
* 获取存储过程详情
*
@@ -263,7 +263,7 @@ public abstract class DbBaseService {
BaseMapper baseMapper = this.getViewAuthBaseMapper(sourceId);
return baseMapper.getProcedureDetail(dbName, typeName, procName);
}
/**
* 删除存储过程
*
@@ -275,7 +275,7 @@ public abstract class DbBaseService {
BaseMapper baseMapper = this.getViewAuthBaseMapper(sourceId);
baseMapper.deleteProcedure(dbName, typeName, procName);
}
/**
* 保存存储过程
*
@@ -288,7 +288,7 @@ public abstract class DbBaseService {
// 需要各数据服务自己实现,各数据库产品的实现都不一样
throw new ConfirmException("暂未支持的数据库类型");
}
/**
* 获取全量数据查询的SQL
*
@@ -305,7 +305,7 @@ public abstract class DbBaseService {
}
return sqlSb.toString();
}
/**
* 获取分页查询的SQL
*
@@ -326,7 +326,22 @@ public abstract class DbBaseService {
sqlSb.append(String.format(" limit %s offset %s", dataViewParam.getPageSize(), dataViewParam.getOffset()));
return sqlSb.toString();
}
/**
* 获取分页查询的SQL
*
* @return 分页查询的SQL
* @author diantu
* @since 2023年2月22日
*/
public String getQueryPageSqlBySql(String sql,Integer pageSize,Integer pageNum) {
StringBuilder sqlSb = new StringBuilder();
sqlSb.append(String.format("select * from (%s) r", sql));
Integer offset = (pageNum-1)*pageSize;
sqlSb.append(String.format(" limit %s offset %s", pageSize, offset));
return sqlSb.toString();
}
/**
* 获取指定数据库的SQL
*
@@ -340,7 +355,7 @@ public abstract class DbBaseService {
}
return null;
}
/**
* 获取查询总条数的SQL
*
@@ -356,7 +371,7 @@ public abstract class DbBaseService {
}
return sqlSb.toString();
}
/**
* 获取表数据
*
@@ -375,7 +390,7 @@ public abstract class DbBaseService {
}
return null;
}
public void deleteTableLineData(Long sourceId, String dbName, String tableName, JSONArray lineJsonArr) {
for (int i = 0; i < lineJsonArr.size(); i++) {
JSONObject lineParam = lineJsonArr.getJSONObject(i);

View File

@@ -45,12 +45,7 @@ public class DmServiceImpl extends DbBaseService {
String[] queryColumnsArray = queryColumns.split(",");
String resultString = "";
for(int i=0;i<queryColumnsArray.length;i++){
if(queryColumnsArray[i].equalsIgnoreCase("IDENTITY")){
queryColumnsArray[i] = "\"IDENTITY\"";
}
if(queryColumnsArray[i].equalsIgnoreCase("DOMAIN")){
queryColumnsArray[i] = "\"DOMAIN\"";
}
queryColumnsArray[i] = "\""+queryColumnsArray[i]+"\"";
if(i < queryColumnsArray.length-1){
resultString +=queryColumnsArray[i] + ",";
}else{

View File

@@ -63,6 +63,22 @@ public class OracleServiceImpl extends DbBaseService {
return sqlSbFinal.toString();
}
/**
* 获取分页查询的SQL
*
* @return 分页查询的SQL
* @author diantu
* @since 2023年2月22日
*/
@Override
public String getQueryPageSqlBySql(String sql,Integer pageSize,Integer pageNum) {
StringBuilder sqlSb = new StringBuilder();
Integer pageSizeFinal = pageSize * pageNum;
Integer pageNumFinal = pageSize * (pageNum - 1) + 1;
sqlSb.append(String.format("select * from ( select r.*,rownum rn from %s", "(" + sql + ") r where rownum<=" + pageSizeFinal + " ) t2 where t2.rn >=" + pageNumFinal));
return sqlSb.toString();
}
/**
* 获取建表语句
*

View File

@@ -98,6 +98,22 @@ public class SqlserverServiceImpl extends DbBaseService {
return sqlSb.toString();
}
/**
* 获取分页查询的SQL(兼容写法,支持SQL Server 2005及以上版本)
*
* @return 分页查询的SQL
* @author diantu
* @since 2023年2月22日
*/
@Override
public String getQueryPageSqlBySql(String sql,Integer pageSize,Integer pageNum) {
StringBuilder sqlSb = new StringBuilder();
Integer rownumber = (pageNum-1)*pageSize;
Integer top = pageSize*pageNum;
sqlSb.append(String.format("select * from ( select row_number()over(order by tempColumn)rownumber,* from (select top %s tempColumn=0,* from (%s) r where 1=1 )a)b where rownumber > %s",top,sql,rownumber));
return sqlSb.toString();
}
/**
* 获取查询总条数的SQL
*

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -1 +1 @@
<!DOCTYPE html><html lang=en><head><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><link rel=icon href=favicon-db.png><title>数据库文档管理</title><link href=css/chunk-vendors.9d3aebdb.css rel=preload as=style><link href=css/index.efa9bf7c.css rel=preload as=style><link href=js/chunk-vendors.97ab6530.js rel=preload as=script><link href=js/index.a92af771.js rel=preload as=script><link href=css/chunk-vendors.9d3aebdb.css rel=stylesheet><link href=css/index.efa9bf7c.css rel=stylesheet></head><body><noscript><strong>We're sorry but zyplayer-db-ui doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id=app></div><script src=js/chunk-vendors.97ab6530.js></script><script src=js/index.a92af771.js></script></body></html>
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="favicon-db.png"><title>数据库文档管理</title><link href="css/chunk-vendors.140b583e.css" rel="preload" as="style"><link href="css/index.0ed31336.css" rel="preload" as="style"><link href="js/chunk-vendors.3dc94571.js" rel="preload" as="script"><link href="js/index.f8a69857.js" rel="preload" as="script"><link href="css/chunk-vendors.140b583e.css" rel="stylesheet"><link href="css/index.0ed31336.css" rel="stylesheet"></head><body><noscript><strong>We're sorry but zyplayer-db-ui doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div><script src="js/chunk-vendors.3dc94571.js"></script><script src="js/index.f8a69857.js"></script></body></html>

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long