41 changed files with 30753 additions and 472 deletions
@ -0,0 +1,58 @@ |
|||||
|
package io.sc.platform.core.util; |
||||
|
|
||||
|
import net.lingala.zip4j.ZipFile; |
||||
|
import net.lingala.zip4j.exception.ZipException; |
||||
|
|
||||
|
import java.io.File; |
||||
|
import java.io.IOException; |
||||
|
|
||||
|
public class ZipUtil { |
||||
|
public static void zip(String targetDirPath) throws IOException { |
||||
|
zip(targetDirPath,targetDirPath + ".zip",false); |
||||
|
} |
||||
|
|
||||
|
public static void zip(String targetDirPath,boolean delete) throws IOException { |
||||
|
zip(targetDirPath,targetDirPath + ".zip",delete); |
||||
|
} |
||||
|
|
||||
|
public static void zip(String targetDirPath,String targetZipFilePath) throws IOException { |
||||
|
zip(targetDirPath,targetZipFilePath,false); |
||||
|
} |
||||
|
|
||||
|
public static void zip(String targetDirPath,String targetZipFilePath,boolean delete) throws IOException { |
||||
|
File targetDirFile =new File(targetDirPath); |
||||
|
if(targetDirFile.exists() && targetDirFile.isDirectory()) { |
||||
|
File targetZipFile =new File(targetZipFilePath); |
||||
|
if(targetZipFile.exists() && targetZipFile.isFile()){ |
||||
|
targetZipFile.delete(); |
||||
|
} |
||||
|
File targetZipFileParent =targetZipFile.getParentFile(); |
||||
|
if(!targetZipFileParent.exists() || !targetZipFileParent.isDirectory()){ |
||||
|
targetZipFileParent.mkdirs(); |
||||
|
} |
||||
|
ZipFile zipFile = new ZipFile(targetZipFilePath); |
||||
|
if (targetDirFile.exists() && targetDirFile.isDirectory()) { |
||||
|
File[] fs = targetDirFile.listFiles(); |
||||
|
if (fs != null && fs.length > 0) { |
||||
|
for (File f : fs) { |
||||
|
if (f.isDirectory()) { |
||||
|
zipFile.addFolder(f); |
||||
|
} else { |
||||
|
zipFile.addFile(f); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
zipFile.close(); |
||||
|
if (delete) { |
||||
|
FileUtil.deldirs(targetDirFile); |
||||
|
} |
||||
|
}else{ |
||||
|
throw new IOException(targetDirPath + " NOT exists or is NOT a directory"); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public static void main(String[] args) throws IOException { |
||||
|
ZipUtil.zip("/Users/wangshaoping/wspsc/workspace/wangshaoping/v8/platform/app.platform/work/web/export/liquibase/platform","/Users/wangshaoping/wspsc/workspace/wangshaoping/v8/platform/app.platform/work/web/export/liquibase2/platform.zip"); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,106 @@ |
|||||
|
<template> |
||||
|
<q-form action="post"> |
||||
|
<div class="row py-1"> |
||||
|
<div class="col-3"></div> |
||||
|
<div class="col-6"> |
||||
|
<w-select |
||||
|
v-model="formDataRef.datasource" |
||||
|
:label="$t('developer.backend.export.liquibase.datasource')" |
||||
|
:options="datasourceOptionsRef" |
||||
|
style="width: 100%" |
||||
|
></w-select> |
||||
|
</div> |
||||
|
<div class="col-3"></div> |
||||
|
</div> |
||||
|
<div class="row py-1"> |
||||
|
<div class="col-3"></div> |
||||
|
<div class="col-6"> |
||||
|
<q-file ref="fileRef" v-model="formDataRef.file" :label="$t('file.multiple.tip')" dense outlined clearable counter accept=".csv"> |
||||
|
<template #prepend> |
||||
|
<q-icon name="cloud_upload" /> |
||||
|
</template> |
||||
|
</q-file> |
||||
|
</div> |
||||
|
<div class="col-3"></div> |
||||
|
</div> |
||||
|
<div class="row py-1"> |
||||
|
<div class="col-3"></div> |
||||
|
<div class="col-6"> |
||||
|
<w-checkbox v-model="formDataRef.deleteFirst" :label="$t('developer.backend.import.liquibase.deleteFirst')" style="width: 100%"></w-checkbox> |
||||
|
</div> |
||||
|
<div class="col-3"></div> |
||||
|
</div> |
||||
|
<div class="row py-1"> |
||||
|
<div class="col-3"></div> |
||||
|
<div class="col-6 row justify-center q-gutter-md py-2"> |
||||
|
<WProgressBtn |
||||
|
ref="progressBtnRef" |
||||
|
icon="bi-database-up" |
||||
|
:label="$t('import')" |
||||
|
data-url="/api/jdbc/data/traceImporterExecuteProgress" |
||||
|
@click="importData" |
||||
|
></WProgressBtn> |
||||
|
</div> |
||||
|
<div class="col-3"></div> |
||||
|
</div> |
||||
|
</q-form> |
||||
|
</template> |
||||
|
<script setup lang="ts"> |
||||
|
import { ref, reactive, onMounted, onUpdated } from 'vue'; |
||||
|
import { useI18n } from 'vue-i18n'; |
||||
|
import { axios, Environment, DialogManager } from 'platform-core'; |
||||
|
import WProgressBtn from './WProgressBtn.vue'; |
||||
|
|
||||
|
const { t } = useI18n(); |
||||
|
|
||||
|
const progressBtnRef = ref(); |
||||
|
const datasourceOptionsRef = ref([]); |
||||
|
const formDataRef = reactive({ |
||||
|
datasource: undefined, |
||||
|
file: undefined, |
||||
|
deleteFirst: false, |
||||
|
}); |
||||
|
const fileRef = ref(); |
||||
|
|
||||
|
const importData = () => { |
||||
|
DialogManager.confirm(t('developer.backend.import.liquibase.import.tip'), () => { |
||||
|
axios |
||||
|
.post( |
||||
|
Environment.apiContextPath('/api/jdbc/data/importData'), |
||||
|
{ |
||||
|
datasource: formDataRef.datasource, |
||||
|
deleteFirst: formDataRef.deleteFirst, |
||||
|
files: fileRef.value.nativeEl.files[0], |
||||
|
}, |
||||
|
{ |
||||
|
headers: { |
||||
|
'Content-Type': 'multipart/form-data', |
||||
|
}, |
||||
|
}, |
||||
|
) |
||||
|
.then(() => { |
||||
|
progressBtnRef.value.start(); |
||||
|
}); |
||||
|
}); |
||||
|
}; |
||||
|
|
||||
|
const loadDatasource = () => { |
||||
|
axios.get(Environment.apiContextPath('/api/system/datasource?pageable=false&sortBy=name')).then((response) => { |
||||
|
const data = response?.data.content; |
||||
|
if (data && data.length > 0) { |
||||
|
datasourceOptionsRef.value.splice(0, datasourceOptionsRef.value.length); |
||||
|
for (let item of data) { |
||||
|
datasourceOptionsRef.value.push({ label: item.name, value: item.name }); |
||||
|
} |
||||
|
} |
||||
|
}); |
||||
|
}; |
||||
|
|
||||
|
onMounted(() => { |
||||
|
loadDatasource(); |
||||
|
}); |
||||
|
|
||||
|
onUpdated(() => { |
||||
|
loadDatasource(); |
||||
|
}); |
||||
|
</script> |
||||
@ -1,242 +0,0 @@ |
|||||
package io.sc.platform.jdbc.liquibase.exporter; |
|
||||
|
|
||||
import io.sc.platform.core.DirectoryManager; |
|
||||
import io.sc.platform.core.Environment; |
|
||||
import io.sc.platform.core.support.ProgressInfo; |
|
||||
import io.sc.platform.core.util.StringUtil; |
|
||||
import io.sc.platform.core.util.WriterUtil; |
|
||||
import io.sc.platform.jdbc.exporter.DataExporter; |
|
||||
import io.sc.platform.jdbc.exporter.support.DataExportConfigure; |
|
||||
import io.sc.platform.jdbc.exporter.support.ExportTable; |
|
||||
import io.sc.platform.jdbc.liquibase.exporter.support.ParentChildRecord; |
|
||||
import io.sc.platform.jdbc.meta.MetaDataLoader; |
|
||||
import io.sc.platform.jdbc.meta.support.Column; |
|
||||
import io.sc.platform.jdbc.meta.support.Table; |
|
||||
import io.sc.platform.jdbc.util.SqlTypeUtil; |
|
||||
import org.slf4j.Logger; |
|
||||
import org.slf4j.LoggerFactory; |
|
||||
import org.springframework.jdbc.datasource.DataSourceUtils; |
|
||||
import org.springframework.jdbc.support.MetaDataAccessException; |
|
||||
|
|
||||
import javax.sql.DataSource; |
|
||||
import javax.xml.bind.DatatypeConverter; |
|
||||
import javax.xml.stream.XMLStreamException; |
|
||||
import javax.xml.stream.XMLStreamWriter; |
|
||||
import java.io.File; |
|
||||
import java.io.FileNotFoundException; |
|
||||
import java.io.UnsupportedEncodingException; |
|
||||
import java.sql.*; |
|
||||
import java.util.ArrayList; |
|
||||
import java.util.List; |
|
||||
import java.util.Locale; |
|
||||
|
|
||||
public class LiquibaseDataExporter implements DataExporter { |
|
||||
private static final Logger log = LoggerFactory.getLogger(LiquibaseDataExporter.class); |
|
||||
private static final String OUTPUT_PATH = DirectoryManager.getInstance().getByName(""); |
|
||||
|
|
||||
@Override |
|
||||
public void export(DataSource dataSource, DataExportConfigure configure, ProgressInfo progressInfo, Locale locale) { |
|
||||
try { |
|
||||
exportSingleFile(dataSource, configure); |
|
||||
// exportMultiFile(dataSource, configure);
|
|
||||
}catch (Exception e){ |
|
||||
throw new RuntimeException(e); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
private void exportSingleFile(DataSource dataSource, DataExportConfigure configure) throws XMLStreamException, FileNotFoundException, UnsupportedEncodingException, MetaDataAccessException { |
|
||||
XMLStreamWriter writer = WriterUtil.xmlStreamWriter(OUTPUT_PATH + File.separator + "data." + configure.getSchema() + ".xml"); |
|
||||
writeHeader(writer); |
|
||||
String[] tableNames =ExportTable.getTableNames(configure.getTables()); |
|
||||
List<Table> tables =MetaDataLoader.newInstance().getTables(dataSource,configure.getSchema(),tableNames); |
|
||||
for(Table table : tables){ |
|
||||
writeTable(dataSource,table,writer); |
|
||||
} |
|
||||
writeTail(writer); |
|
||||
writer.flush(); |
|
||||
writer.close(); |
|
||||
} |
|
||||
|
|
||||
private void exportMultiFile(DataSource dataSource, DataExportConfigure configure) throws MetaDataAccessException, XMLStreamException, FileNotFoundException, UnsupportedEncodingException { |
|
||||
String[] tableNames =ExportTable.getTableNames(configure.getTables()); |
|
||||
List<Table> tables =MetaDataLoader.newInstance().getTables(dataSource,configure.getSchema(),tableNames); |
|
||||
for(Table table : tables){ |
|
||||
XMLStreamWriter writer = WriterUtil.xmlStreamWriter(OUTPUT_PATH + File.separator + "data." + configure.getSchema() + "." + table.getName() + ".xml"); |
|
||||
writeHeader(writer); |
|
||||
writeTable(dataSource,table,writer); |
|
||||
writeTail(writer); |
|
||||
writer.flush(); |
|
||||
writer.close(); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
private void writeHeader(XMLStreamWriter writer) throws XMLStreamException { |
|
||||
writer.writeStartDocument(Environment.DEFAULT_CHARSET_NAME,"1.0"); |
|
||||
writer.writeStartDocument(); |
|
||||
writer.writeStartElement("export"); |
|
||||
} |
|
||||
|
|
||||
private void writeTail(XMLStreamWriter writer) throws XMLStreamException { |
|
||||
writer.writeEndElement(); |
|
||||
} |
|
||||
|
|
||||
private void writeTable(DataSource dataSource,Table table,XMLStreamWriter writer) throws XMLStreamException { |
|
||||
//写入字段定义信息===============================================================================
|
|
||||
List<Column> columns =table.getColumns(); |
|
||||
if(columns==null || columns.size()==0){ |
|
||||
throw new RuntimeException("No column found"); |
|
||||
} |
|
||||
writer.writeStartElement("table"); //start table
|
|
||||
writer.writeAttribute("name", table.getName()); |
|
||||
writer.writeAttribute("remarks", table.getRemarks()); |
|
||||
writer.writeStartElement("sql"); //start sql
|
|
||||
writer.writeCharacters("select * from " + table.getName()); |
|
||||
writer.writeEndElement(); //end sql
|
|
||||
|
|
||||
writer.writeStartElement("fields"); //start fields
|
|
||||
for(int i=0;i<columns.size();i++){ |
|
||||
Column column =columns.get(i); |
|
||||
writer.writeStartElement("field"); //start field
|
|
||||
writer.writeAttribute("name", column.getName()); |
|
||||
writer.writeAttribute("type", "" + column.getJavaType()); |
|
||||
writer.writeAttribute("typeName", column.getSqlType()); |
|
||||
writer.writeAttribute("remarks", column.getRemarks()); |
|
||||
writer.writeEndElement(); //end field
|
|
||||
} |
|
||||
writer.writeEndElement(); //end fields
|
|
||||
|
|
||||
//写入记录集信息==================================================================================
|
|
||||
writeResultSet(dataSource,table,writer); |
|
||||
|
|
||||
writer.writeEndElement(); //end table
|
|
||||
} |
|
||||
|
|
||||
private void writeResultSet(DataSource dataSource,Table table,XMLStreamWriter writer) throws XMLStreamException { |
|
||||
try( |
|
||||
Connection connection = DataSourceUtils.getConnection(dataSource); |
|
||||
Statement statement =connection.createStatement(ResultSet.TYPE_FORWARD_ONLY,ResultSet.CONCUR_READ_ONLY); |
|
||||
){ |
|
||||
statement.setFetchSize(1000); |
|
||||
try(ResultSet rs =statement.executeQuery("select * from " + table.getName());){ |
|
||||
if(table.isSelfReference()){ |
|
||||
log.info("export table [" + table.getName() + "] with parent-and-children relationship: " + table.getSelfReferenceForeignKeyColumnName() + " --> " + table.getSelfReferencePrimaryKeyColumnName()); |
|
||||
exportResultSetWithSelfReference(rs,table,writer); |
|
||||
}else{ |
|
||||
log.info("export table [" + table.getName() + "]"); |
|
||||
exportResultSet(rs,table,writer); |
|
||||
} |
|
||||
}catch (Exception e){ |
|
||||
throw new RuntimeException(e); |
|
||||
} |
|
||||
}catch (Exception e){ |
|
||||
throw new RuntimeException(e); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
private void exportResultSetWithSelfReference(ResultSet rs,Table table,XMLStreamWriter writer) throws XMLStreamException, SQLException, UnsupportedEncodingException { |
|
||||
String pkFieldName =table.getSelfReferencePrimaryKeyColumnName(); |
|
||||
String fkFieldName =table.getSelfReferenceForeignKeyColumnName(); |
|
||||
List<Column> columns =table.getColumns(); |
|
||||
List<ParentChildRecord> records =listResultSet(rs,table.getColumns()); |
|
||||
if(records!=null && records.size()>0){ |
|
||||
List<ParentChildRecord> list =ParentChildRecord.buildTree(records, pkFieldName, fkFieldName); |
|
||||
if(list!=null && list.size()>0){ |
|
||||
writer.writeStartElement("resultset"); //start resultset
|
|
||||
long count =1; |
|
||||
for(ParentChildRecord record : list){ |
|
||||
writer.writeStartElement("row"); //start row
|
|
||||
for(int i=0;i<columns.size();i++){ |
|
||||
Column column =columns.get(i); |
|
||||
String value =record.getField(column.getName()); |
|
||||
|
|
||||
if(value!=null) { |
|
||||
if(StringUtil.validateXml10(value)) { |
|
||||
writer.writeStartElement("f"); //start f
|
|
||||
writer.writeCharacters(value); |
|
||||
writer.writeEndElement(); //end f
|
|
||||
}else { |
|
||||
writer.writeStartElement("f"); //start f
|
|
||||
writer.writeAttribute("base64", "true"); |
|
||||
writer.writeCharacters(DatatypeConverter.printBase64Binary(value.getBytes(Environment.DEFAULT_CHARSET_NAME))); |
|
||||
writer.writeEndElement(); //end f
|
|
||||
} |
|
||||
}else { |
|
||||
writer.writeStartElement("f"); //start f
|
|
||||
writer.writeEndElement(); //end f
|
|
||||
} |
|
||||
} |
|
||||
writer.writeEndElement(); //end row
|
|
||||
if(count%100==0){ |
|
||||
writer.flush(); |
|
||||
} |
|
||||
} |
|
||||
writer.writeEndElement(); //end resultset
|
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
private List<ParentChildRecord> listResultSet(ResultSet rs,List<Column> columns) throws SQLException{ |
|
||||
if(rs!=null){ |
|
||||
List<ParentChildRecord> result =new ArrayList<ParentChildRecord>(); |
|
||||
while(rs.next()){ |
|
||||
ParentChildRecord record =new ParentChildRecord(); |
|
||||
for(int i=0;i<columns.size();i++){ |
|
||||
Column column =columns.get(i); |
|
||||
String fieldName =column.getName(); |
|
||||
String fieldValue =null; |
|
||||
if(Types.BLOB==column.getVendorTypeNumber()){ |
|
||||
Blob blob =rs.getBlob(column.getName()); |
|
||||
if(blob!=null && blob.length()>0){ |
|
||||
if(blob.length()<Integer.MAX_VALUE){ |
|
||||
byte[] bytes =blob.getBytes(1, (int)blob.length()); |
|
||||
fieldValue =DatatypeConverter.printBase64Binary(bytes); |
|
||||
}else{ |
|
||||
throw new RuntimeException(column.getName() + " field is a BLOB, and it's length great than " + Integer.MAX_VALUE + ", can not exported"); |
|
||||
} |
|
||||
} |
|
||||
}else{ |
|
||||
fieldValue =rs.getString(fieldName); |
|
||||
} |
|
||||
record.addField(fieldName, fieldValue); |
|
||||
} |
|
||||
result.add(record); |
|
||||
} |
|
||||
return result; |
|
||||
} |
|
||||
return null; |
|
||||
} |
|
||||
|
|
||||
private void exportResultSet(ResultSet rs,Table table,XMLStreamWriter writer) throws XMLStreamException, SQLException, UnsupportedEncodingException { |
|
||||
writer.writeStartElement("resultset"); //start resultset
|
|
||||
long count =1; |
|
||||
List<Column> columns =table.getColumns(); |
|
||||
while(rs.next()){ |
|
||||
writer.writeStartElement("row"); //start row
|
|
||||
for(int i=0;i<columns.size();i++){ |
|
||||
Column column =columns.get(i); |
|
||||
String fieldName =column.getName(); |
|
||||
String value = SqlTypeUtil.getString(rs,fieldName,column.getVendorTypeNumber()); |
|
||||
if(value!=null) { |
|
||||
if(StringUtil.validateXml10(value)) { |
|
||||
writer.writeStartElement("f"); //start f
|
|
||||
writer.writeCharacters(value); |
|
||||
writer.writeEndElement(); //end f
|
|
||||
}else { |
|
||||
writer.writeStartElement("f"); //start f
|
|
||||
writer.writeAttribute("base64", "true"); |
|
||||
writer.writeCharacters(DatatypeConverter.printBase64Binary(value.getBytes(Environment.DEFAULT_CHARSET_NAME))); |
|
||||
writer.writeEndElement(); //end f
|
|
||||
} |
|
||||
}else { |
|
||||
writer.writeStartElement("f"); //start f
|
|
||||
writer.writeEndElement(); //end f
|
|
||||
} |
|
||||
} |
|
||||
writer.writeEndElement(); //end row
|
|
||||
if(count%1000==0){ |
|
||||
writer.flush(); |
|
||||
} |
|
||||
} |
|
||||
writer.writeEndElement(); //end resultset
|
|
||||
} |
|
||||
} |
|
||||
@ -0,0 +1,55 @@ |
|||||
|
package io.sc.platform.jdbc.liquibase.importer; |
||||
|
|
||||
|
import io.sc.platform.core.DirectoryManager; |
||||
|
import io.sc.platform.core.Environment; |
||||
|
import io.sc.platform.core.support.ProgressInfo; |
||||
|
import io.sc.platform.core.util.DateUtil; |
||||
|
import io.sc.platform.core.util.FileUtil; |
||||
|
import io.sc.platform.core.util.WriterUtil; |
||||
|
import io.sc.platform.core.util.ZipUtil; |
||||
|
import io.sc.platform.jdbc.exporter.DataExporter; |
||||
|
import io.sc.platform.jdbc.exporter.support.DataExportConfigure; |
||||
|
import io.sc.platform.jdbc.exporter.support.DataImportConfigure; |
||||
|
import io.sc.platform.jdbc.exporter.support.ExportTable; |
||||
|
import io.sc.platform.jdbc.importer.DataImporter; |
||||
|
import io.sc.platform.jdbc.liquibase.exporter.support.ParentChildRecord; |
||||
|
import io.sc.platform.jdbc.liquibase.task.CsvImportTaskChange; |
||||
|
import io.sc.platform.jdbc.meta.MetaDataLoader; |
||||
|
import io.sc.platform.jdbc.meta.support.Column; |
||||
|
import io.sc.platform.jdbc.meta.support.Table; |
||||
|
import io.sc.platform.jdbc.util.SqlTypeUtil; |
||||
|
import liquibase.database.jvm.JdbcConnection; |
||||
|
import liquibase.repackaged.com.opencsv.CSVWriterBuilder; |
||||
|
import liquibase.repackaged.com.opencsv.ICSVWriter; |
||||
|
import org.slf4j.Logger; |
||||
|
import org.slf4j.LoggerFactory; |
||||
|
import org.springframework.jdbc.datasource.DataSourceUtils; |
||||
|
import org.springframework.web.multipart.MultipartFile; |
||||
|
|
||||
|
import javax.sql.DataSource; |
||||
|
import java.io.*; |
||||
|
import java.sql.*; |
||||
|
import java.util.Date; |
||||
|
import java.util.*; |
||||
|
import java.util.stream.Collectors; |
||||
|
|
||||
|
public class LiquibaseDataCsvImporter implements DataImporter { |
||||
|
@Override |
||||
|
public void importData(DataSource dataSource, DataImportConfigure configure, ProgressInfo progressInfo, Locale locale) throws Exception { |
||||
|
progressInfo.setRunning(true); |
||||
|
progressInfo.setStartDatetime(new Date()); |
||||
|
progressInfo.setTotalWeight(configure.getFiles().size()+1); |
||||
|
progressInfo.addWeight(1); |
||||
|
|
||||
|
List<MultipartFile> multipartFiles =configure.getFiles(); |
||||
|
if(multipartFiles!=null && !multipartFiles.isEmpty()){ |
||||
|
JdbcConnection jdbcConnection =new JdbcConnection(dataSource.getConnection()); |
||||
|
for(MultipartFile multipartFile : multipartFiles){ |
||||
|
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(multipartFile.getInputStream(), Environment.DEFAULT_CHARSET_NAME)); |
||||
|
CsvImportTaskChange change =new CsvImportTaskChange(); |
||||
|
change.execute(jdbcConnection,bufferedReader,configure.isDeleteFirst()); |
||||
|
} |
||||
|
jdbcConnection.close(); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -1,13 +0,0 @@ |
|||||
{ |
|
||||
"table" : "", |
|
||||
"sql" : "", |
|
||||
"cols": [ |
|
||||
{ |
|
||||
"name": "name", |
|
||||
"remark": "remark", |
|
||||
"javaType": "java.lang.String", |
|
||||
"sqlType": "CBLOB", |
|
||||
"skip": false |
|
||||
} |
|
||||
] |
|
||||
} |
|
||||
|
@ -0,0 +1 @@ |
|||||
|
io.sc.platform.jdbc.liquibase.importer.LiquibaseDataCsvImporter |
||||
@ -0,0 +1,23 @@ |
|||||
|
package io.sc.platform.jdbc.liquibase.task; |
||||
|
|
||||
|
import liquibase.database.jvm.JdbcConnection; |
||||
|
|
||||
|
import java.sql.Connection; |
||||
|
import java.sql.DriverManager; |
||||
|
|
||||
|
public class CsvImportTaskChangeTest { |
||||
|
public static void main(String[] args) throws Exception { |
||||
|
String url ="jdbc:p6spy:mysql://localhost:3306/platform?autoReconnect=true&allowPublicKeyRetrieval=true&useSSL=false&useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai"; |
||||
|
String username ="platform"; |
||||
|
String password ="platform"; |
||||
|
Connection connection = DriverManager.getConnection(url,username,password); |
||||
|
JdbcConnection jdbcConnection =new JdbcConnection(connection); |
||||
|
|
||||
|
CsvImportTaskChange change =new CsvImportTaskChange(); |
||||
|
change.setDataFile("classpath:/mv_score_record.csv"); |
||||
|
change.execute(jdbcConnection); |
||||
|
jdbcConnection.close(); |
||||
|
connection.close(); |
||||
|
System.out.println("completed"); |
||||
|
} |
||||
|
} |
||||
File diff suppressed because it is too large
@ -0,0 +1,90 @@ |
|||||
|
package io.sc.platform.jdbc.controller; |
||||
|
|
||||
|
import io.sc.platform.core.support.ProgressInfo; |
||||
|
import io.sc.platform.jdbc.controller.support.ExporterThread; |
||||
|
import io.sc.platform.jdbc.controller.support.ImporterThread; |
||||
|
import io.sc.platform.jdbc.exporter.exception.ExporterRunningException; |
||||
|
import io.sc.platform.jdbc.exporter.support.DataExportConfigure; |
||||
|
import io.sc.platform.jdbc.exporter.support.DataImportConfigure; |
||||
|
import io.sc.platform.jdbc.service.JdbcDataService; |
||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||
|
import org.springframework.web.bind.annotation.*; |
||||
|
import org.springframework.web.multipart.MultipartFile; |
||||
|
|
||||
|
import java.io.InputStream; |
||||
|
import java.util.Locale; |
||||
|
|
||||
|
@RestController |
||||
|
@RequestMapping("/api/jdbc/data") |
||||
|
public class JdbcDataWebController { |
||||
|
@Autowired private JdbcDataService jdbcDataService; |
||||
|
private ImporterThread importerThread =null; |
||||
|
private ExporterThread exporterThread =null; |
||||
|
|
||||
|
@PostMapping("importData") |
||||
|
public void importData( |
||||
|
@RequestParam(name="datasource",required = false) String datasource, |
||||
|
@RequestParam(name="deleteFirst", required = false) boolean deleteFirst, |
||||
|
@RequestPart(name="files",required = false) MultipartFile multipartFile, |
||||
|
Locale locale) throws Exception |
||||
|
{ |
||||
|
//检查有当前用户启动的执行线程是否正在运行
|
||||
|
if(importerThread!=null && importerThread.isAlive()){ |
||||
|
throw new ExporterRunningException(); |
||||
|
} |
||||
|
DataImportConfigure configure =new DataImportConfigure(); |
||||
|
configure.setDatasource(datasource); |
||||
|
configure.setDeleteFirst(deleteFirst); |
||||
|
configure.addFile(multipartFile); |
||||
|
//启动新线程
|
||||
|
importerThread =new ImporterThread(jdbcDataService,configure,locale); |
||||
|
importerThread.start(); |
||||
|
} |
||||
|
|
||||
|
@PostMapping("exportData") |
||||
|
public void exportData(@RequestBody DataExportConfigure configure, Locale locale) throws Exception { |
||||
|
//检查有当前用户启动的执行线程是否正在运行
|
||||
|
if(exporterThread!=null && exporterThread.isAlive()){ |
||||
|
throw new ExporterRunningException(); |
||||
|
} |
||||
|
//启动新线程
|
||||
|
exporterThread =new ExporterThread(jdbcDataService,configure,locale); |
||||
|
exporterThread.start(); |
||||
|
} |
||||
|
|
||||
|
@GetMapping("traceImporterExecuteProgress") |
||||
|
public ProgressInfo traceImporterExecuteProgress(Locale locale) throws Exception{ |
||||
|
if(importerThread!=null){ |
||||
|
Exception exception =importerThread.getException(); |
||||
|
if(exception!=null){ |
||||
|
importerThread.setException(null); |
||||
|
throw exception; |
||||
|
} |
||||
|
if(importerThread.isAlive()){ |
||||
|
if(importerThread.getException()!=null){ |
||||
|
throw importerThread.getException(); |
||||
|
} |
||||
|
return importerThread.getProgressInfo(); |
||||
|
} |
||||
|
} |
||||
|
return ProgressInfo.notRunning(); |
||||
|
} |
||||
|
|
||||
|
@GetMapping("traceExporterExecuteProgress") |
||||
|
public ProgressInfo traceExporterExecuteProgress(Locale locale) throws Exception{ |
||||
|
if(exporterThread!=null){ |
||||
|
Exception exception =exporterThread.getException(); |
||||
|
if(exception!=null){ |
||||
|
exporterThread.setException(null); |
||||
|
throw exception; |
||||
|
} |
||||
|
if(exporterThread.isAlive()){ |
||||
|
if(exporterThread.getException()!=null){ |
||||
|
throw exporterThread.getException(); |
||||
|
} |
||||
|
return exporterThread.getProgressInfo(); |
||||
|
} |
||||
|
} |
||||
|
return ProgressInfo.notRunning(); |
||||
|
} |
||||
|
} |
||||
@ -1,43 +0,0 @@ |
|||||
package io.sc.platform.jdbc.controller; |
|
||||
|
|
||||
import io.sc.platform.core.support.ProgressInfo; |
|
||||
import io.sc.platform.jdbc.controller.support.ExporterThread; |
|
||||
import io.sc.platform.jdbc.exporter.exception.ExporterNotRunningException; |
|
||||
import io.sc.platform.jdbc.exporter.exception.ExporterRunningException; |
|
||||
import io.sc.platform.jdbc.exporter.support.DataExportConfigure; |
|
||||
import io.sc.platform.jdbc.service.JdbcExportService; |
|
||||
import org.springframework.beans.factory.annotation.Autowired; |
|
||||
import org.springframework.web.bind.annotation.*; |
|
||||
|
|
||||
import java.util.Date; |
|
||||
import java.util.Locale; |
|
||||
|
|
||||
@RestController |
|
||||
@RequestMapping("/api/jdbc/export") |
|
||||
public class JdbcExportWebController { |
|
||||
@Autowired private JdbcExportService jdbcExportService; |
|
||||
private ExporterThread thread =null; |
|
||||
|
|
||||
@PostMapping("export") |
|
||||
public void export(@RequestBody DataExportConfigure configure, Locale locale) throws Exception { |
|
||||
//检查有当前用户启动的执行线程是否正在运行
|
|
||||
if(thread!=null && thread.isAlive()){ |
|
||||
throw new ExporterRunningException(); |
|
||||
} |
|
||||
//启动新线程
|
|
||||
thread =new ExporterThread(jdbcExportService,configure,locale); |
|
||||
thread.start(); |
|
||||
} |
|
||||
|
|
||||
@GetMapping("traceExecuteProgress") |
|
||||
public ProgressInfo traceExecuteProgress(Locale locale) throws Exception{ |
|
||||
if(thread!=null && thread.isAlive()){ |
|
||||
if(thread.getException()!=null){ |
|
||||
throw thread.getException(); |
|
||||
} |
|
||||
return thread.getProgressInfo(); |
|
||||
}else{ |
|
||||
return ProgressInfo.notRunning(); |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
@ -0,0 +1,44 @@ |
|||||
|
package io.sc.platform.jdbc.controller.support; |
||||
|
|
||||
|
import io.sc.platform.core.support.ProgressInfo; |
||||
|
import io.sc.platform.jdbc.exporter.support.DataExportConfigure; |
||||
|
import io.sc.platform.jdbc.exporter.support.DataImportConfigure; |
||||
|
import io.sc.platform.jdbc.service.JdbcDataService; |
||||
|
import org.slf4j.Logger; |
||||
|
import org.slf4j.LoggerFactory; |
||||
|
|
||||
|
import java.util.Locale; |
||||
|
|
||||
|
public class ImporterThread extends Thread { |
||||
|
private Logger log = LoggerFactory.getLogger(ImporterThread.class); |
||||
|
private JdbcDataService jdbcDataService; |
||||
|
private DataImportConfigure configure; |
||||
|
private Locale locale; |
||||
|
private volatile ProgressInfo progressInfo =new ProgressInfo(100,0); |
||||
|
private Exception exception; |
||||
|
|
||||
|
public ImporterThread(JdbcDataService jdbcDataService, DataImportConfigure configure, Locale locale){ |
||||
|
this.jdbcDataService =jdbcDataService; |
||||
|
this.configure =configure; |
||||
|
this.locale =locale; |
||||
|
} |
||||
|
@Override |
||||
|
public void run() { |
||||
|
try { |
||||
|
jdbcDataService.importData(configure,progressInfo,locale); |
||||
|
} catch (Exception e) { |
||||
|
this.exception =e; |
||||
|
log.error("",e); |
||||
|
} |
||||
|
} |
||||
|
public ProgressInfo getProgressInfo(){ |
||||
|
return progressInfo; |
||||
|
} |
||||
|
public Exception getException() { |
||||
|
return exception; |
||||
|
} |
||||
|
|
||||
|
public void setException(Exception exception) { |
||||
|
this.exception = exception; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,51 @@ |
|||||
|
package io.sc.platform.jdbc.exporter.support; |
||||
|
|
||||
|
import org.springframework.web.multipart.MultipartFile; |
||||
|
|
||||
|
import java.util.ArrayList; |
||||
|
import java.util.List; |
||||
|
|
||||
|
public class DataImportConfigure { |
||||
|
private String datasource; |
||||
|
private boolean deleteFirst; |
||||
|
private List<MultipartFile> files =new ArrayList<>(); |
||||
|
|
||||
|
public DataImportConfigure addFile(MultipartFile file){ |
||||
|
this.files.add(file); |
||||
|
return this; |
||||
|
} |
||||
|
|
||||
|
|
||||
|
public String getDatasource() { |
||||
|
return datasource; |
||||
|
} |
||||
|
|
||||
|
public void setDatasource(String datasource) { |
||||
|
this.datasource = datasource; |
||||
|
} |
||||
|
|
||||
|
public boolean isDeleteFirst() { |
||||
|
return deleteFirst; |
||||
|
} |
||||
|
|
||||
|
public void setDeleteFirst(boolean deleteFirst) { |
||||
|
this.deleteFirst = deleteFirst; |
||||
|
} |
||||
|
|
||||
|
public List<MultipartFile> getFiles() { |
||||
|
return files; |
||||
|
} |
||||
|
|
||||
|
public void setFiles(List<MultipartFile> files) { |
||||
|
this.files = files; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public String toString() { |
||||
|
return "DataImportConfigure{" + |
||||
|
"datasource='" + datasource + '\'' + |
||||
|
", deleteFirst=" + deleteFirst + |
||||
|
", files=" + files + |
||||
|
'}'; |
||||
|
} |
||||
|
} |
||||
@ -1,8 +1,37 @@ |
|||||
package io.sc.platform.jdbc.importer; |
package io.sc.platform.jdbc.importer; |
||||
|
|
||||
|
import io.sc.platform.core.support.ProgressInfo; |
||||
|
import io.sc.platform.jdbc.exporter.DataExporter; |
||||
|
import io.sc.platform.jdbc.exporter.support.DataExportConfigure; |
||||
|
import io.sc.platform.jdbc.exporter.support.DataImportConfigure; |
||||
|
|
||||
import javax.sql.DataSource; |
import javax.sql.DataSource; |
||||
import java.io.Reader; |
import java.io.Reader; |
||||
|
import java.util.Iterator; |
||||
|
import java.util.Locale; |
||||
|
import java.util.ServiceLoader; |
||||
|
|
||||
public interface DataImporter { |
public interface DataImporter { |
||||
public void importFrom(DataSource dataSource, String schemaName, Reader reader); |
public void importData(DataSource dataSource, DataImportConfigure configure, ProgressInfo progressInfo, Locale locale) throws Exception; |
||||
|
|
||||
|
public static DataImporter newInstance(){ |
||||
|
ServiceLoader<DataImporter> serviceLoader =ServiceLoader.load(DataImporter.class); |
||||
|
Iterator<DataImporter> services =serviceLoader.iterator(); |
||||
|
while(services.hasNext()){ |
||||
|
return services.next(); |
||||
|
} |
||||
|
return null; |
||||
|
} |
||||
|
|
||||
|
public static DataImporter newInstance(String implementsClass){ |
||||
|
ServiceLoader<DataImporter> serviceLoader =ServiceLoader.load(DataImporter.class); |
||||
|
Iterator<DataImporter> services =serviceLoader.iterator(); |
||||
|
while(services.hasNext()){ |
||||
|
DataImporter importer =services.next(); |
||||
|
if(importer.getClass().getName().equals(implementsClass)){ |
||||
|
return importer; |
||||
|
} |
||||
|
} |
||||
|
return null; |
||||
|
} |
||||
} |
} |
||||
|
|||||
@ -0,0 +1,14 @@ |
|||||
|
package io.sc.platform.jdbc.service; |
||||
|
|
||||
|
import io.sc.platform.core.support.ProgressInfo; |
||||
|
import io.sc.platform.jdbc.exporter.support.DataExportConfigure; |
||||
|
import io.sc.platform.jdbc.exporter.support.DataImportConfigure; |
||||
|
import org.springframework.web.bind.annotation.RequestParam; |
||||
|
import org.springframework.web.multipart.MultipartFile; |
||||
|
|
||||
|
import java.util.Locale; |
||||
|
|
||||
|
public interface JdbcDataService { |
||||
|
public void importData(DataImportConfigure configure, ProgressInfo progressInfo, Locale locale) throws Exception; |
||||
|
public void exportData(DataExportConfigure configure, ProgressInfo progressInfo, Locale locale) throws Exception; |
||||
|
} |
||||
@ -1,10 +0,0 @@ |
|||||
package io.sc.platform.jdbc.service; |
|
||||
|
|
||||
import io.sc.platform.core.support.ProgressInfo; |
|
||||
import io.sc.platform.jdbc.exporter.support.DataExportConfigure; |
|
||||
|
|
||||
import java.util.Locale; |
|
||||
|
|
||||
public interface JdbcExportService { |
|
||||
public void export(DataExportConfigure configure, ProgressInfo progressInfo, Locale locale) throws Exception; |
|
||||
} |
|
||||
@ -0,0 +1,54 @@ |
|||||
|
package io.sc.platform.jdbc.service.impl; |
||||
|
|
||||
|
import io.sc.platform.core.Environment; |
||||
|
import io.sc.platform.core.support.ProgressInfo; |
||||
|
import io.sc.platform.jdbc.exporter.DataExporter; |
||||
|
import io.sc.platform.jdbc.exporter.support.DataExportConfigure; |
||||
|
import io.sc.platform.jdbc.exporter.support.DataImportConfigure; |
||||
|
import io.sc.platform.jdbc.importer.DataImporter; |
||||
|
import io.sc.platform.jdbc.service.DatasourceService; |
||||
|
import io.sc.platform.jdbc.service.JdbcDataService; |
||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.springframework.util.StringUtils; |
||||
|
import org.springframework.web.multipart.MultipartFile; |
||||
|
|
||||
|
import javax.sql.DataSource; |
||||
|
import java.io.BufferedReader; |
||||
|
import java.io.InputStreamReader; |
||||
|
import java.util.Date; |
||||
|
import java.util.List; |
||||
|
import java.util.Locale; |
||||
|
|
||||
|
@Service |
||||
|
public class JdbcDataServiceImpl implements JdbcDataService { |
||||
|
@Autowired private DatasourceService datasourceService; |
||||
|
|
||||
|
@Override |
||||
|
public void importData(DataImportConfigure configure, ProgressInfo progressInfo, Locale locale) throws Exception { |
||||
|
if(configure==null){ |
||||
|
return; |
||||
|
} |
||||
|
DataSource dataSource =null; |
||||
|
if(StringUtils.hasText(configure.getDatasource())){ |
||||
|
dataSource =datasourceService.getDatasource(configure.getDatasource()); |
||||
|
}else{ |
||||
|
dataSource =datasourceService.getDefaultDatasource(); |
||||
|
} |
||||
|
DataImporter.newInstance().importData(dataSource,configure,progressInfo,locale); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void exportData(DataExportConfigure configure, ProgressInfo progressInfo, Locale locale) throws Exception { |
||||
|
if(configure==null){ |
||||
|
return; |
||||
|
} |
||||
|
DataSource dataSource =null; |
||||
|
if(StringUtils.hasText(configure.getDatasource())){ |
||||
|
dataSource =datasourceService.getDatasource(configure.getDatasource()); |
||||
|
}else{ |
||||
|
dataSource =datasourceService.getDefaultDatasource(); |
||||
|
} |
||||
|
DataExporter.newInstance().exportData(dataSource,configure,progressInfo,locale); |
||||
|
} |
||||
|
} |
||||
@ -1,43 +0,0 @@ |
|||||
package io.sc.platform.jdbc.service.impl; |
|
||||
|
|
||||
import io.sc.platform.core.support.ProgressInfo; |
|
||||
import io.sc.platform.jdbc.exporter.DataExporter; |
|
||||
import io.sc.platform.jdbc.exporter.support.DataExportConfigure; |
|
||||
import io.sc.platform.jdbc.service.DatasourceService; |
|
||||
import io.sc.platform.jdbc.service.JdbcExportService; |
|
||||
import org.springframework.beans.factory.annotation.Autowired; |
|
||||
import org.springframework.stereotype.Service; |
|
||||
import org.springframework.util.StringUtils; |
|
||||
|
|
||||
import javax.sql.DataSource; |
|
||||
import java.util.Date; |
|
||||
import java.util.Locale; |
|
||||
|
|
||||
@Service |
|
||||
public class JdbcExportServiceImpl implements JdbcExportService { |
|
||||
@Autowired private DatasourceService datasourceService; |
|
||||
|
|
||||
@Override |
|
||||
public void export(DataExportConfigure configure, ProgressInfo progressInfo, Locale locale) throws Exception { |
|
||||
if(configure==null){ |
|
||||
return; |
|
||||
} |
|
||||
progressInfo.setRunning(true); |
|
||||
DataSource dataSource =null; |
|
||||
if(StringUtils.hasText(configure.getDatasource())){ |
|
||||
dataSource =datasourceService.getDatasource(configure.getDatasource()); |
|
||||
}else{ |
|
||||
dataSource =datasourceService.getDefaultDatasource(); |
|
||||
} |
|
||||
|
|
||||
progressInfo.setStartDatetime(new Date()); |
|
||||
progressInfo.setTotalWeight(100); |
|
||||
|
|
||||
DataExporter dataExporter =DataExporter.newInstance(); |
|
||||
dataExporter.export(dataSource,configure,progressInfo,locale); |
|
||||
|
|
||||
//执行完毕
|
|
||||
progressInfo.setCompletedDatetime(new Date()); |
|
||||
progressInfo.setCurrentWeight(progressInfo.getTotalWeight()); |
|
||||
} |
|
||||
} |
|
||||
Loading…
Reference in new issue