manager/zyplayer-doc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

数据库文档oracle支持优化，数据互导雏形

Browse Source
This commit is contained in:
暮光：城中城
2019-09-29 22:05:35 +08:00
parent 9c00186f88
commit 85bad69784
8 changed files with 354 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/controller/DbSqlExecutorController.java
View File

@@ -16,6 +16,7 @@ import com.zyplayer.doc.data.repository.support.consts.DocAuthConst;
import com.zyplayer.doc.data.service.manage.DbFavoriteService;
import com.zyplayer.doc.data.service.manage.DbHistoryService;
import com.zyplayer.doc.db.framework.consts.DbAuthType;
import com.zyplayer.doc.db.framework.db.mapper.base.ExecuteParam;
import com.zyplayer.doc.db.framework.db.mapper.base.ExecuteResult;
import com.zyplayer.doc.db.framework.db.mapper.base.ExecuteType;
import com.zyplayer.doc.db.framework.db.mapper.base.SqlExecutor;
@@ -69,9 +70,13 @@ public class DbSqlExecutorController {
}
sqlItem = sqlItem.trim();
try {
Map<String, Object> paramMap = JSON.parseObject(params);
ExecuteType executeType = (!manageAuth && select) ? ExecuteType.SELECT : ExecuteType.ALL;
ExecuteResult executeResult = sqlExecutor.execute(sourceId, executeId, executeType, sqlItem, paramMap);
ExecuteParam executeParam = new ExecuteParam();
executeParam.setDatasourceId(sourceId);
executeParam.setExecuteId(executeId);
executeParam.setExecuteType(executeType);
executeParam.setSql(sqlItem);
ExecuteResult executeResult = sqlExecutor.execute(executeParam);
SerializeConfig mapping = new SerializeConfig();
mapping.put(Date.class, new SimpleDateFormatSerializer("yyyy-MM-dd HH:mm:ss"));
mapping.put(Timestamp.class, new SimpleDateFormatSerializer("yyyy-MM-dd HH:mm:ss"));

3
zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/framework/configuration/DatasourceUtil.java
View File

@@ -38,6 +38,9 @@ public class DatasourceUtil {
dataSource.setConnectionErrorRetryAttempts(2);
dataSource.setBreakAfterAcquireFailure(true);
dataSource.setName("zyplayer-doc-db-" + dbDatasource.getId());
if (dbDatasource.getSourceUrl().contains("oracle")) {
dataSource.setValidationQuery("select 1 from dual");
}
DruidPooledConnection connection = dataSource.getConnection(3000);
if (connection == null) {
throw new ConfirmException("尝试获取该数据源连接失败：" + dbDatasource.getSourceUrl());

68
zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/framework/db/mapper/base/ExecuteParam.java Normal file
View File

@@ -0,0 +1,68 @@
package com.zyplayer.doc.db.framework.db.mapper.base;
import org.apache.ibatis.mapping.ParameterMapping;
import java.util.List;
/**
* 通过SQL和参数列表处理后的执行参数对象
*
* @author 暮光：城中城
* @since 2019-09-28
*/
public class ExecuteParam {
private String sql;
private List<Object> paramList;
private List<ParameterMapping> parameterMappings;
private Long datasourceId;
private String executeId;
private ExecuteType executeType;
public Long getDatasourceId() {
return datasourceId;
}
public void setDatasourceId(Long datasourceId) {
this.datasourceId = datasourceId;
}
public String getExecuteId() {
return executeId;
}
public void setExecuteId(String executeId) {
this.executeId = executeId;
}
public ExecuteType getExecuteType() {
return executeType;
}
public void setExecuteType(ExecuteType executeType) {
this.executeType = executeType;
}
public List<ParameterMapping> getParameterMappings() {
return parameterMappings;
}
public void setParameterMappings(List<ParameterMapping> parameterMappings) {
this.parameterMappings = parameterMappings;
}
public String getSql() {
return sql;
}
public void setSql(String sql) {
this.sql = sql;
}
public List<Object> getParamList() {
return paramList;
}
public void setParamList(List<Object> paramList) {
this.paramList = paramList;
}
}

33
zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/framework/db/mapper/base/SqlExecutor.java
View File

@@ -8,6 +8,7 @@ import com.zyplayer.doc.db.framework.db.bean.DatabaseRegistrationBean;
import org.apache.ibatis.builder.SqlSourceBuilder;
import org.apache.ibatis.builder.StaticSqlSource;
import org.apache.ibatis.mapping.BoundSql;
import org.apache.ibatis.mapping.ParameterMapping;
import org.apache.ibatis.parsing.GenericTokenParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -60,9 +61,9 @@ public class SqlExecutor {
* @author 暮光：城中城
* @since 2019年8月18日
*/
public ExecuteResult execute(Long datasourceId, String executeId, ExecuteType executeType, String sql, Map<String, Object> paramMap) {
DatabaseFactoryBean factoryBean = databaseRegistrationBean.getFactoryById(datasourceId);
return this.execute(factoryBean, executeId, executeType, sql, paramMap, null);
public ExecuteResult execute(ExecuteParam param) {
DatabaseFactoryBean factoryBean = databaseRegistrationBean.getFactoryById(param.getDatasourceId());
return this.execute(factoryBean, param, null);
}
/**
@@ -70,14 +71,14 @@ public class SqlExecutor {
* @author 暮光：城中城
* @since 2019年8月18日
*/
public ExecuteResult execute(DatabaseFactoryBean factoryBean, String executeId, ExecuteType executeType, String sqlStr, Map<String, Object> paramMap, ResultHandler handler) {
public ExecuteResult execute(DatabaseFactoryBean factoryBean, ExecuteParam executeParam, ResultHandler handler) {
if (factoryBean == null) {
return ExecuteResult.error("未找到数据库连接", sqlStr);
return ExecuteResult.error("未找到数据库连接", executeParam.getSql());
}
// BoundSql boundSql = getBoundSql(sql, paramMap);
// sql = boundSql.getSql();
// String sqlStr = SqlLogUtil.getSqlString(paramMap, boundSql);
logger.info("sql ==> {}", sqlStr);
logger.info("sql ==> {}", executeParam.getSql());
// List<ParameterMapping> parameterMappings = boundSql.getParameterMappings();
PreparedStatement preparedStatement = null;
@@ -86,15 +87,19 @@ public class SqlExecutor {
try {
long startTime = System.currentTimeMillis();
connection = factoryBean.getDataSource().getConnection();
preparedStatement = connection.prepareStatement(sqlStr);
preparedStatement = connection.prepareStatement(executeParam.getSql());
// 设置当前的PreparedStatement
statementMap.put(executeId, preparedStatement);
// for (int i = 0; i < parameterMappings.size(); i++) {
// preparedStatement.setObject(i + 1, paramMap.get(parameterMappings.get(i).getProperty()));
// }
statementMap.put(executeParam.getExecuteId(), preparedStatement);
List<ParameterMapping> parameterMappings = executeParam.getParameterMappings();
List<Object> paramDataList = executeParam.getParamList();
if (parameterMappings != null && paramDataList != null && parameterMappings.size() > 0 && paramDataList.size() > 0) {
for (int i = 0; i < parameterMappings.size(); i++) {
preparedStatement.setObject(i + 1, paramDataList.get(i));
}
}
// 限制下最大数量
preparedStatement.setMaxRows(1000);
if (ExecuteType.SELECT.equals(executeType)) {
if (ExecuteType.SELECT.equals(executeParam.getExecuteType())) {
preparedStatement.executeQuery();
} else {
preparedStatement.execute();
@@ -119,11 +124,11 @@ public class SqlExecutor {
// 更新的数量，小于0代表不是更新语句
int updateCount = preparedStatement.getUpdateCount();
long useTime = System.currentTimeMillis() - startTime;
return new ExecuteResult(updateCount, resultList, useTime, sqlStr);
return new ExecuteResult(updateCount, resultList, useTime, executeParam.getSql());
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
statementMap.remove(executeId);
statementMap.remove(executeParam.getExecuteId());
try {
if (preparedStatement != null && !preparedStatement.isClosed()) {
preparedStatement.close();

27
zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/framework/db/mapper/oracle/OracleBaseMapper.xml
View File

@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.zyplayer.doc.db.framework.db.mapper.base.BaseMapper">
<resultMap id="TableColumnDescDtoMap" type="com.zyplayer.doc.db.framework.db.dto.TableColumnDescDto" >
<result column="TABLE_NAME" property="tableName" jdbcType="VARCHAR" />
<result column="COLUMN_NAME" property="name" jdbcType="VARCHAR" />
@@ -9,7 +9,7 @@
<result column="NULLABLE" property="nullable" jdbcType="VARCHAR" />
<result column="COMMENTS" property="description" jdbcType="VARCHAR" />
</resultMap>
<resultMap id="QueryTableColumnDescDtoMap" type="com.zyplayer.doc.db.framework.db.dto.QueryTableColumnDescDto" >
<result column="TABLE_NAME" property="tableName" jdbcType="VARCHAR" />
<result column="COLUMN_NAME" property="columnName" jdbcType="VARCHAR" />
@@ -18,46 +18,43 @@
<!-- 获取用户列表 -->
<select id="getDatabaseList" resultType="com.zyplayer.doc.db.framework.db.dto.DatabaseInfoDto">
select USERNAME dbName db from all_users
select USERNAME dbName from all_users
</select>
<!-- 获取表列表 -->
<select id="getTableList" resultType="com.zyplayer.doc.db.framework.db.dto.TableInfoDto">
select t.OWNER as dbName,t.TABLE_NAME as tableName,c.COMMENTS as tableComment from all_tables t left join user_tab_comments c on t.TABLE_NAME = c.TABLE_NAME
<where>
<if test="dbName != null"> where t.owner = #{dbName}</if>
<if test="dbName != null">t.owner = #{dbName}</if>
</where>
</select>
<!-- 获取表字段集合 -->
<select id="getTableColumnList" resultMap="TableColumnDescDtoMap">
select t.TABLE_NAME,t.COLUMN_NAME,t.DATA_TYPE,case t.NULLABLE when 'Y' then 1 when 'N' then 0 end NULLABLE, c.COMMENTS
select t.TABLE_NAME,t.COLUMN_NAME,t.DATA_TYPE,case t.NULLABLE when 'Y' then 1 when 'N' then 0 end NULLABLE, c.COMMENTS
from user_tab_columns t left join user_col_comments c on t.COLUMN_NAME = c.COLUMN_NAME and t.TABLE_NAME = c.TABLE_NAME
<where>
t.table_name in (select table_name from all_tables where owner = #{dbName} )
<if test="tableName != null"> and t.TABLE_NAME = #{tableName}</if>
</where>
</select>
<!-- 条件查询表字段 -->
<select id="getTableAndColumnBySearch" resultMap="QueryTableColumnDescDtoMap">
select t.TABLE_NAME,t.COLUMN_NAME,t.DATA_TYPE,case t.NULLABLE when 'Y' then 1 when 'N' then 0 end NULLABLE, c.COMMENTS
from user_tab_columns t
left join user_col_comments c on t.COLUMN_NAME = c.COLUMN_NAME and t.TABLE_NAME = c.TABLE_NAME
<where>
<if test="tableName != null">and t.TABLE_NAME = #{tableName}</if>
AND (t.COLUMN_NAME like #{searchText} or c.COMMENTS like #{searchText})
</where>
where t.TABLE_NAME = #{tableName} AND (t.COLUMN_NAME like #{searchText} or c.COMMENTS like #{searchText})
</select>
<!-- 条件查询表 -->
<select id="getTableDescList" resultType="com.zyplayer.doc.db.framework.db.dto.TableDescDto">
select t.OWNER,t.TABLE_NAME as tableName,c.COMMENTS as description
from all_tables t
left join user_tab_comments c on t.TABLE_NAME = c.TABLE_NAME
<where>
t.owner = #{owner}
<if test="tableName != null">and t.TABLE_NAME = #{tableName}</if>
<if test="dbName != null">and t.owner = #{dbName}</if>
<if test="tableName != null">and t.TABLE_NAME = #{tableName}</if>
</where>
</select>

184
zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/framework/db/transfer/SqlParseUtil.java Normal file
View File

@@ -0,0 +1,184 @@
package com.zyplayer.doc.db.framework.db.transfer;
import com.zyplayer.doc.db.framework.db.mapper.base.ExecuteParam;
import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
import net.sf.jsqlparser.expression.operators.relational.ItemsList;
import net.sf.jsqlparser.parser.CCJSqlParserManager;
import net.sf.jsqlparser.schema.Column;
import net.sf.jsqlparser.statement.Statement;
import net.sf.jsqlparser.statement.insert.Insert;
import net.sf.jsqlparser.statement.select.PlainSelect;
import org.apache.ibatis.builder.SqlSourceBuilder;
import org.apache.ibatis.builder.StaticSqlSource;
import org.apache.ibatis.mapping.BoundSql;
import org.apache.ibatis.mapping.ParameterMapping;
import org.apache.ibatis.session.Configuration;
import org.springframework.util.CollectionUtils;
import java.io.StringReader;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
public class SqlParseUtil {
/**
* 通过SQL和参数列表获取处理后的SQL和参数
* @param sql
* @param paramList
* @return
*/
public static List<ExecuteParam> getExecuteParamList(String sql, List<Map<String, Object>> paramList) {
// 单条操作
if (paramList == null || paramList.size() <= 1) {
Map<String, Object> paramMap = CollectionUtils.isEmpty(paramList) ? Collections.emptyMap() : paramList.get(0);
ExecuteParam executeParam = getSingleExecuteParam(sql, paramMap);
return Collections.singletonList(executeParam);
}
// 批量的insert语法
ExecuteParam multiExecuteParam = getMultiExecuteParam(sql, paramList);
if (multiExecuteParam != null) {
return Collections.singletonList(multiExecuteParam);
}
// 不是insert语法单条处理
List<ExecuteParam> executeParamList = new LinkedList<>();
for (Map<String, Object> paramMap : paramList) {
ExecuteParam executeParam = getSingleExecuteParam(sql, paramMap);
executeParamList.add(executeParam);
}
return executeParamList;
}
/**
* 解析sql，如果是insert则把values拼成多个，用于支持批量插入
* 不是insert则返回null
* 不支持insert set语法，不支持values以外使用#{column}语法
* @param sql
* @param paramList
* @return
*/
public static ExecuteParam getMultiExecuteParam(String sql, List<Map<String, Object>> paramList) {
List<Object> resultParamList = new LinkedList<>();
SqlSourceBuilder sqlSourceBuilder = new SqlSourceBuilder(new Configuration());
String insertSql = sql.replaceAll("(#\\{\\w+})", "'$1'");
// 处理${column}参数
Insert insert = getInsertStmt(insertSql);
if (insert == null) {
return null;
}
StringBuilder sqlBuilder = new StringBuilder();
sqlBuilder.append("INSERT ");
if (insert.getModifierPriority() != null) {
sqlBuilder.append(insert.getModifierPriority().name()).append(" ");
}
if (insert.isModifierIgnore()) {
sqlBuilder.append("IGNORE ");
}
sqlBuilder.append("INTO ");
sqlBuilder.append(insert.getTable()).append(" ");
if (insert.getColumns() != null) {
sqlBuilder.append(PlainSelect.getStringList(insert.getColumns(), true, true)).append(" ");
}
if (insert.isUseValues()) {
sqlBuilder.append("VALUES ");
}
if (insert.getItemsList() != null) {
ExpressionList expressionList = (ExpressionList) insert.getItemsList();
String fillSql = expressionList.toString().replaceAll("'(#\\{\\w+})'", "$1");
StringBuilder itemsSb = new StringBuilder();
for (Map<String, Object> paramMap : paramList) {
if (itemsSb.length() > 0) {
itemsSb.append(", ");
}
itemsSb.append(fillSql);
// 解析获取参数
StaticSqlSource parse = (StaticSqlSource) sqlSourceBuilder.parse(fillSql, Object.class, Collections.emptyMap());
BoundSql boundSql = parse.getBoundSql(new Object());
List<ParameterMapping> parameterMappings = boundSql.getParameterMappings();
for (ParameterMapping parameterMapping : parameterMappings) {
resultParamList.add(paramMap.get(parameterMapping.getProperty()));
}
}
sqlBuilder.append(itemsSb.toString());
}
// pagehelper 使用了jsqlparser包，而且版本很低，，低版本没这个方法
// if (insert.isUseSet()) {
// throw new RuntimeException("暂不支持insert set语法");
// }
if (insert.isUseDuplicate()) {
sqlBuilder.append(" ON DUPLICATE KEY UPDATE ");
List<Column> duplicateUpdateColumns = insert.getDuplicateUpdateColumns();
for (int i = 0; i < duplicateUpdateColumns.size(); i++) {
if (i != 0) {
sqlBuilder.append(", ");
}
sqlBuilder.append(duplicateUpdateColumns.get(i)).append(" = ");
sqlBuilder.append(insert.getDuplicateUpdateExpressionList().get(i));
}
}
if (insert.isReturningAllColumns()) {
sqlBuilder.append(" RETURNING *");
} else if (insert.getReturningExpressionList() != null) {
sqlBuilder.append(" RETURNING ").append(PlainSelect.getStringList(insert.getReturningExpressionList(), true, false));
}
StaticSqlSource parse = (StaticSqlSource) sqlSourceBuilder.parse(sqlBuilder.toString(), Object.class, Collections.emptyMap());
BoundSql boundSql = parse.getBoundSql(new Object());
List<ParameterMapping> parameterMappings = boundSql.getParameterMappings();
// values 里面的参数个数和所需参数个数不一致，则不支持批量，说明values外面还存在#{column}的参数
if (parameterMappings.size() != resultParamList.size()) {
throw new RuntimeException("insert除values以外不支持#{column}的动态参数");
}
// 组装结果
ExecuteParam executeParam = new ExecuteParam();
executeParam.setParameterMappings(parameterMappings);
executeParam.setSql(boundSql.getSql());
executeParam.setParamList(resultParamList);
return executeParam;
}
/**
* 解析单条操作的SQL
* @param sql
* @param paramMap
* @return
*/
public static ExecuteParam getSingleExecuteParam(String sql, Map<String, Object> paramMap) {
ExecuteParam executeParam = new ExecuteParam();
SqlSourceBuilder sqlSourceBuilder = new SqlSourceBuilder(new Configuration());
StaticSqlSource parse = (StaticSqlSource) sqlSourceBuilder.parse(sql, Object.class, paramMap);
BoundSql boundSql = parse.getBoundSql(new Object());
List<ParameterMapping> parameterMappings = boundSql.getParameterMappings();
List<Object> resultParamList = new LinkedList<>();
for (ParameterMapping parameterMapping : parameterMappings) {
resultParamList.add(paramMap.get(parameterMapping.getProperty()));
}
executeParam.setSql(boundSql.getSql());
executeParam.setParameterMappings(parameterMappings);
executeParam.setParamList(resultParamList);
return executeParam;
}
/**
* 获取sql语句里面查询的列
*
* @return
*/
public static Insert getInsertStmt(String sql) {
try {
CCJSqlParserManager parser = new CCJSqlParserManager();
Statement stmt = parser.parse(new StringReader(sql));
if (stmt instanceof Insert) {
Insert insertStmt = (Insert) stmt;
ItemsList itemsList = insertStmt.getItemsList();
if (itemsList instanceof ExpressionList) {
return insertStmt;
}
}
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
}

55
zyplayer-doc-db/src/main/java/com/zyplayer/doc/db/framework/db/transfer/TransferDataServer.java Normal file
View File

@@ -0,0 +1,55 @@
package com.zyplayer.doc.db.framework.db.transfer;
import cn.hutool.core.util.RandomUtil;
import com.zyplayer.doc.db.framework.db.bean.DatabaseFactoryBean;
import com.zyplayer.doc.db.framework.db.bean.DatabaseRegistrationBean;
import com.zyplayer.doc.db.framework.db.mapper.base.ExecuteParam;
import com.zyplayer.doc.db.framework.db.mapper.base.ExecuteResult;
import com.zyplayer.doc.db.framework.db.mapper.base.ExecuteType;
import com.zyplayer.doc.db.framework.db.mapper.base.SqlExecutor;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@Service
public class TransferDataServer {
@Resource
SqlExecutor sqlExecutor;
@Resource
DatabaseRegistrationBean databaseRegistrationBean;
// 批量插入一批次的条数
private final Integer batchInsertNum = 100;
public void writeData(Long querySourceId, Long storageSourceId, String querySql, String storageSql) {
List<Map<String, Object>> selectResultList = new LinkedList<>();
DatabaseFactoryBean factoryBean = databaseRegistrationBean.getFactoryById(querySourceId);
ExecuteParam executeParam = new ExecuteParam();
executeParam.setSql(querySql);
executeParam.setExecuteType(ExecuteType.SELECT);
executeParam.setExecuteId(RandomUtil.simpleUUID());
ExecuteResult executeResult = sqlExecutor.execute(factoryBean, executeParam, resultMap -> {
selectResultList.add(resultMap);
// 批量输出数据
if (selectResultList.size() >= batchInsertNum) {
this.writeData(storageSourceId, storageSql, selectResultList);
}
});
// 不足100的数据
if (selectResultList.size() > 0) {
this.writeData(storageSourceId, storageSql, selectResultList);
}
}
public void writeData(Long storageSourceId, String storageSql, List<Map<String, Object>> selectResultList) {
List<ExecuteParam> executeParamList = SqlParseUtil.getExecuteParamList(storageSql, selectResultList);
for (ExecuteParam executeParam : executeParamList) {
executeParam.setDatasourceId(storageSourceId);
sqlExecutor.execute(executeParam);
}
selectResultList.clear();
}
}

11
zyplayer-doc-manage/pom.xml
View File

@@ -73,11 +73,12 @@
<artifactId>zyplayer-doc-data</artifactId>
<version>${zyplayer.doc.version}</version>
</dependency>
<dependency>
<groupId>com.zyplayer</groupId>
<artifactId>zyplayer-doc-grpc</artifactId>
<version>${zyplayer.doc.version}</version>
</dependency>
<!--zyplayer-doc-grpc 默认不加载进来-->
<!-- <dependency>-->
<!-- <groupId>com.zyplayer</groupId>-->
<!-- <artifactId>zyplayer-doc-grpc</artifactId>-->
<!-- <version>${zyplayer.doc.version}</version>-->
<!-- </dependency>-->
<!--zyplayer-doc-swagger-->
<dependency>
<groupId>com.zyplayer</groupId>