manager/zyplayer-doc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改SQL执行逻辑，解决一个sql中字段名重复取值混乱问题，优化SQL编辑器

Browse Source
This commit is contained in:
暮光：城中城
2023-02-15 20:35:34 +08:00
parent 6aec6ce8d6
commit 852727b3ad
9 changed files with 479 additions and 14126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/controller/DbSqlExecutorController.java
View File

@@ -1,9 +1,13 @@
package com.zyplayer.doc.db.controller;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
import com.zyplayer.doc.core.annotation.AuthMan;
import com.zyplayer.doc.core.exception.ConfirmException;
import com.zyplayer.doc.data.config.security.DocUserDetails;
import com.zyplayer.doc.data.config.security.DocUserUtil;
import com.zyplayer.doc.data.repository.manage.entity.DbFavorite;
@@ -14,10 +18,7 @@ import com.zyplayer.doc.data.repository.support.consts.DocSysType;
import com.zyplayer.doc.data.service.manage.DbFavoriteService;
import com.zyplayer.doc.data.service.manage.DbHistoryService;
import com.zyplayer.doc.db.framework.consts.DbAuthType;
import com.zyplayer.doc.db.framework.db.mapper.base.ExecuteParam;
import com.zyplayer.doc.db.framework.db.mapper.base.ExecuteResult;
import com.zyplayer.doc.db.framework.db.mapper.base.ExecuteType;
import com.zyplayer.doc.db.framework.db.mapper.base.SqlExecutor;
import com.zyplayer.doc.db.framework.db.mapper.base.*;
import com.zyplayer.doc.db.framework.db.transfer.SqlParseUtil;
import com.zyplayer.doc.db.framework.json.DocDbResponseJson;
import com.zyplayer.doc.db.framework.utils.JSONUtil;
@@ -47,7 +48,7 @@ public class DbSqlExecutorController {
private static Logger logger = LoggerFactory.getLogger(DbSqlExecutorController.class);
@Resource
SqlExecutor sqlExecutor;
ColumnSqlExecutor columnSqlExecutor;
@Resource
DbHistoryService dbHistoryService;
@Resource
@@ -72,44 +73,44 @@ public class DbSqlExecutorController {
dbHistoryService.saveHistory(sql.trim(), params, sourceId);
// 参数处理
Map<String, Object> paramMap = JSON.parseObject(params);
List<String> resultList = new LinkedList<>();
// 支持;分割的多个sql执行
String[] sqlArr = sql.split(";");
// 执行条数太多，反应慢，展示结果栏太多，也不应该在这一次执行很多条语句，应该使用导入
if (sqlArr.length > 20) {
return DocDbResponseJson.warn("单次执行最多支持20条语句同时执行，当前语句条数：" + sqlArr.length);
}
for (String sqlItem : sqlArr) {
if (StringUtils.isBlank(sqlItem)) {
continue;
// 解析出多个执行的SQL
List<String> analysisQuerySqlList = new LinkedList<>();
try {
List<SQLStatement> sqlStatements = new MySqlStatementParser(sql).parseStatementList();
for (SQLStatement sqlStatement : sqlStatements) {
analysisQuerySqlList.add(sqlStatement.toString());
}
sqlItem = sqlItem.trim();
ExecuteResult executeResult;
ExecuteParam executeParam = new ExecuteParam();
} catch (Exception e) {
return DocDbResponseJson.warn("SQL解析失败：" + e.getMessage());
}
// 执行条数太多，反应慢，展示结果栏太多，也不应该在这一次执行很多条语句，应该使用导入
if (analysisQuerySqlList.size() > 20) {
return DocDbResponseJson.warn("单次执行最多支持20条语句同时执行，当前语句条数：" + analysisQuerySqlList.size());
}
List<ColumnExecuteResult> resultList = new LinkedList<>();
for (String singleSql : analysisQuerySqlList) {
ColumnExecuteResult executeResult;
try {
ExecuteType executeType = (manageAuth || update) ? ExecuteType.ALL : ExecuteType.SELECT;
executeParam = SqlParseUtil.getSingleExecuteParam(sqlItem, paramMap);
ExecuteParam executeParam = SqlParseUtil.getSingleExecuteParam(singleSql, paramMap);
executeParam.setDatasourceId(sourceId);
executeParam.setExecuteId(executeId);
executeParam.setExecuteType(executeType);
executeParam.setPrefixSql(useDbSql);
executeParam.setMaxRows(1000);
executeResult = sqlExecutor.execute(executeParam);
executeResult = columnSqlExecutor.execute(executeParam);
} catch (Exception e) {
logger.error("执行出错", e);
executeResult = ExecuteResult.error(e.getMessage(), sqlItem);
executeResult = ColumnExecuteResult.error(singleSql, e.getMessage(), e);
}
// 执行的sql处理
String executeSqlLog = SqlLogUtil.parseLogSql(executeParam.getSql(), executeParam.getParameterMappings(), executeParam.getParamList());
executeResult.setSql(executeSqlLog);
resultList.add(JSON.toJSONString(executeResult, JSONUtil.serializeConfig, SerializerFeature.WriteMapNullValue));
resultList.add(executeResult);
}
return DocDbResponseJson.ok(resultList);
}
@PostMapping(value = "/cancel")
public DocDbResponseJson cancel(String executeId) {
sqlExecutor.cancel(executeId);
columnSqlExecutor.cancel(executeId);
return DocDbResponseJson.ok();
}

79
zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/framework/db/mapper/base/ColumnExecuteResult.java Normal file
View File

@@ -0,0 +1,79 @@
package com.zyplayer.doc.db.framework.db.mapper.base;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Data;
import java.util.List;
/**
* SQL执行结果对象
*
* @author 暮光：城中城
* @since 2019-08-19
*/
@Data
public class ColumnExecuteResult {
private Long queryTime;
private Integer errCode;
private String errMsg;
private Exception exception;
private String executeSql;
private Integer updateCount;
// 查询结果表头
private List<String> header;
// 查询结果数据
private List<List<Object>> data;
public ColumnExecuteResult() {
this.updateCount = 0;
}
public static ColumnExecuteResult ok(String sql) {
ColumnExecuteResult result = new ColumnExecuteResult();
result.setExecuteSql(sql);
result.setErrCode(ExecuteResultCode.SUCCESS);
return result;
}
public static ColumnExecuteResult warn(String sql, String errMsg) {
ColumnExecuteResult result = new ColumnExecuteResult();
result.setExecuteSql(sql);
result.setErrMsg(errMsg);
result.setErrCode(ExecuteResultCode.WARN);
return result;
}
public static ColumnExecuteResult error(String sql, String errMsg, Exception e) {
ColumnExecuteResult result = new ColumnExecuteResult();
result.setExecuteSql(sql);
result.setErrMsg(errMsg);
result.setException(e);
result.setErrCode(ExecuteResultCode.ERROR);
return result;
}
public static class ExecuteResultCode {
public static Integer SUCCESS = 0;
public static Integer WARN = -1;
public static Integer ERROR = -2;
}
/**
* 判断错误并抛出异常
*/
public ColumnExecuteResult judgeAndThrow() {
if (ExecuteResultCode.SUCCESS.equals(errCode)) {
return this;
}
if (ExecuteResultCode.WARN.equals(errCode)) {
throw new RuntimeException(errMsg);
}
if (ExecuteResultCode.ERROR.equals(errCode)) {
if (exception != null) {
throw new RuntimeException(exception);
}
throw new RuntimeException(errMsg);
}
return this;
}
}

173
zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/framework/db/mapper/base/ColumnSqlExecutor.java Normal file
View File

@@ -0,0 +1,173 @@
package com.zyplayer.doc.db.framework.db.mapper.base;
import cn.hutool.core.io.IoUtil;
import com.alibaba.druid.pool.DruidPooledConnection;
import com.zyplayer.doc.db.framework.db.bean.DatabaseFactoryBean;
import com.zyplayer.doc.db.framework.db.bean.DatabaseRegistrationBean;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.ibatis.mapping.ParameterMapping;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Repository;
import javax.annotation.Resource;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* 新版的SQL执行类，返回结果有比较大的差异，为了兼容之前的代码就不在原有基础上修改
*
* @author 暮光：城中城
* @since 2023-02-12
*/
@Repository
public class ColumnSqlExecutor {
private static Logger logger = LoggerFactory.getLogger(SqlExecutor.class);
@Resource
DatabaseRegistrationBean databaseRegistrationBean;
// 执行中的PreparedStatement信息，用于强制取消执行
private static final Map<String, PreparedStatement> statementMap = new ConcurrentHashMap<>();
/**
* 取消执行
*
* @author 暮光：城中城
* @since 2019年8月18日
*/
public boolean cancel(String executeId) {
PreparedStatement preparedStatement = statementMap.remove(executeId);
try {
if (preparedStatement != null) {
preparedStatement.cancel();
return true;
}
logger.error("未找到指定任务，取消执行失败：{}", executeId);
} catch (Exception e) {
e.printStackTrace();
}
return false;
}
/**
* 执行sql，返回结果
*
* @author 暮光：城中城
* @since 2019年8月18日
*/
public ColumnExecuteResult execute(ExecuteParam param) {
DatabaseFactoryBean factoryBean = databaseRegistrationBean.getOrCreateFactoryById(param.getDatasourceId());
return this.execute(factoryBean, param, null);
}
/**
* 执行sql，返回结果
*
* @author 暮光：城中城
* @since 2019年8月18日
*/
public ColumnExecuteResult execute(ExecuteParam param, ResultHandler handler) {
DatabaseFactoryBean factoryBean = databaseRegistrationBean.getOrCreateFactoryById(param.getDatasourceId());
return this.execute(factoryBean, param, handler);
}
/**
* 执行sql，可通过handler回调每一行的结果
*
* @author 暮光：城中城
* @since 2019年8月18日
*/
public ColumnExecuteResult execute(DatabaseFactoryBean factoryBean, ExecuteParam executeParam, ResultHandler handler) {
// 有参数的时候不输出日志，暂时只有导数据才有参数
if (CollectionUtils.isEmpty(executeParam.getParamList())) {
if (StringUtils.isNotBlank(executeParam.getPrefixSql())) {
logger.info("prefix sql ==> {}", executeParam.getPrefixSql());
}
logger.info("sql ==> {}", executeParam.getSql());
}
if (factoryBean == null) {
return ColumnExecuteResult.error(executeParam.getSql(), "未找到数据库连接", null);
}
long startExecuteTime = System.currentTimeMillis();
ColumnExecuteResult executeResult = ColumnExecuteResult.ok(executeParam.getSql());
PreparedStatement preparedStatement = null;
PreparedStatement prefixStatement = null;
DruidPooledConnection connection = null;
ResultSet resultSet = null;
// 执行查询
try {
connection = factoryBean.getDataSource().getConnection();
if (StringUtils.isNotBlank(executeParam.getPrefixSql())) {
prefixStatement = connection.prepareStatement(executeParam.getPrefixSql());
prefixStatement.execute();
}
preparedStatement = connection.prepareStatement(executeParam.getSql());
// 设置当前的PreparedStatement
statementMap.put(executeParam.getExecuteId(), preparedStatement);
List<ParameterMapping> parameterMappings = executeParam.getParameterMappings();
List<Object> paramDataList = executeParam.getParamList();
if (parameterMappings.size() > 0 && paramDataList.size() > 0) {
int parameterCount = 99999;
try {
parameterCount = preparedStatement.getParameterMetaData().getParameterCount();
} catch (Exception e) {
logger.info("不支持获取总数：" + e.getMessage());
}
for (int paramIndex = 0; paramIndex < parameterMappings.size() && paramIndex < parameterCount; paramIndex++) {
preparedStatement.setObject(paramIndex + 1, paramDataList.get(paramIndex));
}
}
// 最大限制1分钟
preparedStatement.setQueryTimeout(60);
// 限制下最大数量
if (executeParam.getMaxRows() != null) {
preparedStatement.setMaxRows(executeParam.getMaxRows());
}
if (ExecuteType.SELECT.equals(executeParam.getExecuteType())) {
preparedStatement.executeQuery();
} else {
preparedStatement.execute();
}
// 查询的结果集
resultSet = preparedStatement.getResultSet();
List<String> headerList = new LinkedList<>();
List<List<Object>> dataList = new LinkedList<>();
if (resultSet != null) {
int columnCount = resultSet.getMetaData().getColumnCount();
for (int i = 0; i < columnCount; i++) {
headerList.add(resultSet.getMetaData().getColumnLabel(i + 1));
}
while (resultSet.next()) {
List<Object> tempList = new LinkedList<>();
for (int i = 0; i < columnCount; i++) {
tempList.add(resultSet.getObject(i + 1));
}
dataList.add(tempList);
}
}
// 更新的数量
executeResult.setData(dataList);
executeResult.setHeader(headerList);
executeResult.setUpdateCount(Math.max(preparedStatement.getUpdateCount(), 0));
} catch (Exception e) {
logger.error("执行出错", e);
executeResult.setException(e);
executeResult.setErrMsg(e.getMessage());
executeResult.setErrCode(ColumnExecuteResult.ExecuteResultCode.ERROR);
} finally {
statementMap.remove(executeParam.getExecuteId());
IoUtil.close(resultSet);
IoUtil.close(prefixStatement);
IoUtil.close(preparedStatement);
IoUtil.close(connection);
}
executeResult.setQueryTime(System.currentTimeMillis() - startExecuteTime);
return executeResult;
}
}

29
zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/framework/db/mapper/base/SqlExecutor.java
View File

@@ -1,5 +1,6 @@
package com.zyplayer.doc.db.framework.db.mapper.base;
import cn.hutool.core.io.IoUtil;
import com.alibaba.druid.pool.DruidPooledConnection;
import com.zyplayer.doc.db.framework.db.bean.DatabaseFactoryBean;
import com.zyplayer.doc.db.framework.db.bean.DatabaseRegistrationBean;
@@ -98,6 +99,7 @@ public class SqlExecutor {
PreparedStatement preparedStatement = null;
PreparedStatement prefixStatement = null;
DruidPooledConnection connection = null;
ResultSet resultSet = null;
// 执行查询
try {
long startTime = System.currentTimeMillis();
@@ -128,7 +130,7 @@ public class SqlExecutor {
preparedStatement.execute();
}
// 查询的结果集
ResultSet resultSet = preparedStatement.getResultSet();
resultSet = preparedStatement.getResultSet();
List<Map<String, Object>> resultList = new LinkedList<>();
if (resultSet != null) {
while (resultSet.next()) {
@@ -153,27 +155,10 @@ public class SqlExecutor {
throw new RuntimeException(e);
} finally {
statementMap.remove(executeParam.getExecuteId());
try {
if (prefixStatement != null && !prefixStatement.isClosed()) {
prefixStatement.close();
}
} catch (Exception e) {
logger.error("关闭Statement失败");
}
try {
if (preparedStatement != null && !preparedStatement.isClosed()) {
preparedStatement.close();
}
} catch (Exception e) {
logger.error("关闭Statement失败");
}
try {
if (connection != null && !connection.isClosed()) {
connection.recycle();
}
} catch (Exception e) {
logger.error("回收connection失败");
}
IoUtil.close(resultSet);
IoUtil.close(prefixStatement);
IoUtil.close(preparedStatement);
IoUtil.close(connection);
}
}
}