報表的操作（一）

easyexcel入門

easyexcel工具類，github地址：https://github.com/alibaba/easyexcel

一、gitHub摘抄，上面demo都有
Java解析、生成Excel比較有名的框架有Apache poi、jxl。但他們都存在一個嚴重的問題就是非常的耗內存，poi有一套SAX模式的API可以一定程度的解決一些內存溢出的問題，但POI還是有一些缺陷，比如07版Excel解壓縮以及解壓後存儲都是在內存中完成的，內存消耗依然很大。easyexcel重寫了poi對07版Excel的解析，能夠原本一個3M的excel用POI sax依然需要100M左右內存降低到幾M，並且再大的excel不會出現內存溢出，03版依賴POI的sax模式。在上層做了模型轉換的封裝，讓使用者更加簡單方便。

二、使用easyexcel工具+sql動態讀取，生成複雜一點的數字報表

1、入口

package com.xxx.xxxx.tool.execl.controller;

import com.alibaba.excel.support.ExcelTypeEnum;
import com.xxx.xxxx.tool.execl.util.ExcelReportTool;
import com.xxx.xxxx.tool.execl.util.SendMailUtil;
import com.xxx.xxxx.tool.execl.util.SqlUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Configurable;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.web.bind.annotation.*;

import javax.servlet.http.HttpServletRequest;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.sql.SQLException;
import java.time.LocalDateTime;

/**
 * @author xxx
 */
@Slf4j
@Configurable
@EnableScheduling
@RestController
@RequestMapping("/report")
public class ExcelReportToolTime {

  private static SqlUtil sqlUtil;

  @Autowired
  private SendMailUtil sendMailUtil;

  private static String urlFile="jdbc:mysql://10.1.5.75:3306/xxxx";
  private static String nameFile="xxxx";
  private static String passwordFile="xxxx";
  private static String thirdFile="xxxx-tools/xxxx-tools-xlsxReportEngine/src/main/resources/input/sql_template_31.xlsx";
  private static String FourthFile="xxxx-tools/xxxx-tools-xlsxReportEngine/src/main/resources/output/result.xls";
  private static String fifthFile="xxxx-tools/xxxx-tools-xlsxReportEngine/src/main/resources/input/export_template_31.xls";
  /**
   * //@param args 0-數據庫連接url 1-賬戶 2-密碼 3-導出sql腳本模板 4-導出文件地址 5-導出填充模板文件
   * */
  public void mainExport() {//String[] args
    try (FileInputStream exportRuleTemplateFile = new FileInputStream(thirdFile);
      FileOutputStream exportFile = new FileOutputStream(FourthFile)) {
      createSqlUtil(urlFile, nameFile, passwordFile);
      ExcelReportTool excelReportTool = new ExcelReportTool();
      excelReportTool.setSqlUtil(sqlUtil);
      if (thirdFile.toUpperCase().endsWith("XLS")) {
        excelReportTool.exportExcel(fifthFile, exportRuleTemplateFile, exportFile, ExcelTypeEnum.XLS);
      }
      if (thirdFile.toUpperCase().endsWith("XLSX")) {
        excelReportTool.exportExcel(fifthFile, exportRuleTemplateFile, exportFile, ExcelTypeEnum.XLSX);
      }
    } catch (FileNotFoundException e) {
      e.printStackTrace();
    } catch (IOException e) {
      e.printStackTrace();
    } catch (SQLException e) {
      e.printStackTrace();
    } finally {
      try {
        sqlUtil.closeConnection();
      } catch (SQLException e) {
        e.printStackTrace();
      }
    }
  }

  private static void createSqlUtil(String url, String username, String password) throws SQLException{
    sqlUtil = new SqlUtil(url, username, password);
  }

  @PostMapping(path = "/excelReportDay")
  public String excelReportDay( @RequestBody String requestMessage, HttpServletRequest request) {
    System.out.println(requestMessage);
    mainExport();
    return "success";
  }

  /**
   * 直接指定時間間隔，例如：每天15：09
   * "0 10 0 1 * ?"  每月1號的0:10:00執行
   * "0 15 10 L * ?" 每月最後一日的上午10:15觸發
   */
  @Scheduled(cron = "0 13 17 * * ?")
  private void configureTasks() {
    log.info("輸出報表數據，執行靜態定時任務時間:{}", LocalDateTime.now());
    mainExport();
    //發送郵件
    sendMailAttach();
  }

  private void sendMailAttach(){
    String to="[email protected]";
    String subject="subject";
    String content="2020月份優惠券統計日報，請查收";
    String filePath=FourthFile;
    //SendMailUtil sendMailUtil = new SendMailUtil();
    sendMailUtil.sendAttachmentMail(to,subject,content,filePath);
  }
}

2、使用的工具類

package com.xxxx.xxxx.tool.execl.util;

import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.read.builder.ExcelReaderBuilder;
import com.alibaba.excel.support.ExcelTypeEnum;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.excel.write.metadata.fill.FillWrapper;
import com.xxxx.xxxx.tool.execl.bean.CommandRuleEnum;
import com.xxxx.xxxx.tool.execl.bean.CommandTypeEnum;
import com.xxxx.xxxx.tool.execl.bean.DataType;
import com.xxxx.xxxx.tool.execl.bean.InputRowModel;
import lombok.Setter;
import org.apache.commons.collections4.CollectionUtils;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.sql.SQLException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;

@Setter
public class ExcelReportTool {

  private SqlUtil sqlUtil;

  private LinkedBlockingQueue linkedBlockingQueue;

  private ThreadPoolExecutor threadPool;

  /**
   * @param templateFile 導出填充模板
   * @param exportRuleTemplateFile 導出腳本模板
   * @param exportFile 導出目標文件
   * @param excelType excel類型 xls xlsx
   * */
  public void exportExcel(String templateFile, FileInputStream exportRuleTemplateFile, FileOutputStream exportFile, ExcelTypeEnum excelType) {
    linkedBlockingQueue = new LinkedBlockingQueue<Runnable>();
    threadPool= new ThreadPoolExecutor(10, 15, 60, TimeUnit.SECONDS, linkedBlockingQueue);
    try {
      ExcelReaderBuilder excelReaderBuilder = EasyExcel.read(exportRuleTemplateFile, InputRowModel.class, null);
      excelReaderBuilder.excelType(excelType);
      List<InputRowModel> list = excelReaderBuilder.sheet(0).doReadSync();
      // 合併命令
      List<InputRowModel> mergeCommand = new LinkedList<>();
      // 待導出list數據
      Map<String, List<Map<String, String>>> listData = new ConcurrentHashMap<>();
      // 待導出map數據(多條sql結果不可有重複字符)
      Map<String, String> mapData = new HashMap<>();
      if (!CollectionUtils.isEmpty(list)) {
        for (InputRowModel row: list) {
          if (row.getCommand().startsWith(CommandTypeEnum.SQL.getType())) {
            exceSql(listData, mapData, row);
            continue;
          }
          if (row.getCommand().startsWith(CommandTypeEnum.MERGE.getType())) {
            mergeCommand.add(row);
          }
        }
      }
      // 等待子線程執行完畢
      threadPool.shutdown();
      while (!threadPool.isTerminated()) {
        Thread.sleep(1000);
      }
      export(batchExceMerge(mergeCommand, listData), mapData, exportFile, templateFile);
    } catch (InterruptedException e) {
      e.printStackTrace();
    } finally {
      try {
        sqlUtil.closeConnection();
      } catch (SQLException e) {
        e.printStackTrace();
      }
    }
  }

  private void exceSql(Map<String, List<Map<String, String>>> listData, Map<String, String> mapData, InputRowModel row) {
    threadPool.execute(new Runnable() {
      @Override
      public void run() {
        String sql = row.getCommand().replaceAll(CommandTypeEnum.SQL.getType(),"");
        try {
          if (row.getType().equals(DataType.LIST.getType())) {
            // 查詢多條輸出List<Map<String, String>>
            listData.put(row.getAlias(), sqlUtil.queryList(sql));
          }
          if (row.getType().equals(DataType.MAP.getType())) {
            // 查詢單條輸出Map<String, String>
            mapData.putAll(sqlUtil.queryMap(sql));
          }
        } catch (SQLException e) {
          e.printStackTrace();
        }
      }
    });
  }

  /**
   * 導出
   * @param exportListResults 帶到處的list數據
   * @param exportMapResults 待導出的map數據
   * @param exportPath 導出文件流
   * */
  private void export(Map<String, List<Map<String, String>>> exportListResults, Map<String, String> exportMapResults, FileOutputStream exportPath, String templateFile) {
    try  {
      ExcelWriter excelWriter = EasyExcel.write(exportPath).withTemplate(templateFile).build();
      WriteSheet writeSheet = EasyExcel.writerSheet().build();
      // 填充list列表
      Iterator<Map.Entry<String, List<Map<String, String>>>> iterator = exportListResults.entrySet().iterator();
      while (iterator.hasNext()) {
        Map.Entry<String, List<Map<String, String>>> row = iterator.next();
        excelWriter.fill(new FillWrapper(row.getKey(), row.getValue()), writeSheet);
      }
      // 填充map
      if(0<exportMapResults.size()){
        excelWriter.fill(exportMapResults, writeSheet);
      }
      excelWriter.finish();
      exportPath.flush();
    } catch (FileNotFoundException e) {
      e.printStackTrace();
    } catch (IOException e) {
      e.printStackTrace();
    }
  }

  /**
   * sql執行結束後執行合併命令
   * @param mergeCommand 待執行合併命令
   * @param data 待合併數據
   * */
  private Map<String, List<Map<String, String>>> batchExceMerge(List<InputRowModel> mergeCommand, Map<String, List<Map<String, String>>> data) {
    Map<String, List<Map<String, String>>> exportResults = new HashMap<>();
    String command;
    for (InputRowModel row: mergeCommand) {
      command = row.getCommand().replaceAll(CommandTypeEnum.MERGE.getType(),"");
      for (CommandRuleEnum value : CommandRuleEnum.values()) {
        if (Pattern.compile(value.getRule()).matcher(command).find()) {
          List<Map<String, String>> merge = exec(value, command, data);
          // 合併後數據存入待導出數據
          exportResults.put(row.getAlias(), merge);
          // 合併後數據存入待合併數據(別的合併命令可能用上？)
          data.put(row.getAlias(), merge);
          break;
        }
      }
    }
    return exportResults;
  }

  /**
   * 執行合併命令
   * @param commandRuleEnum 合併命令類型
   * @param command 合併命令
   * @param data 待合併數據
   * */
  private List<Map<String, String>> exec(CommandRuleEnum commandRuleEnum, String command, Map<String, List<Map<String, String>>> data) {
    List<Map<String, String>> result = null;
    switch (commandRuleEnum.getOperation()) {
      // CommandRuleEnum.MERGE_BY_KEY 根據指定字段匹配合並
      case 1:
        String[] commands = command.split("=");
        result = mergeByKey(data, commands[0], commands[1].split(","));
        break;
      // CommandRuleEnum.MERGE 直接合並
      case 2:
        // TODO
        ;break;
    }
    return result;
  }

  /**
   * 根據指定字段匹配合並
   * @param data 待合併數據
   * @param key 指定匹配字段
   * @param targetData 命令中指定需要合併的數據別名
   */
  private List<Map<String, String>> mergeByKey(Map<String, List<Map<String, String>>> data, String key, String[] targetData) {
    List<Map<String, String>> mergeResult = new LinkedList<>();
    Map<String, Map<String, String>> equalList=new LinkedHashMap<>();
    // 根據別名取出下標爲0的數據整理成K/V map(以指定匹配字段值做Key)，便於匹配合並
    data.get(targetData[0]).forEach(x -> {
      equalList.put(x.get(key), x);
    });
    // 從下標1開始時遍歷與equalList中數據進行匹配合並
    for (int i = 1;i<targetData.length;i++) {
      for (int j = 0;j<data.get(targetData[i]).size();j++) {
        if (equalList.containsKey(data.get(targetData[i]).get(j).get(key))) {
          // 找到匹配數據合併
          equalList.get(data.get(targetData[i]).get(j).get(key)).putAll(data.get(targetData[i]).get(j));
        } else {
          // 未找到匹配數據，新起一條寫入
          equalList.put(data.get(targetData[i]).get(j).get(key), data.get(targetData[i]).get(j));
        }
      }
    }
    equalList.entrySet().forEach(x -> {
      mergeResult.add(x.getValue());
    });
    return mergeResult;
  }

}

數據庫鏈接操作工具

package com.xxxx.xxxx.tool.execl.util;

import java.sql.*;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

public class SqlUtil {

  private Connection connection;

  public SqlUtil(String url, String username, String password) throws SQLException {
    connection = DriverManager.getConnection(url, username, password);
  }

  /**
   * 多行查詢輸出List<Map<String, String>>
   * */
  public List<Map<String, String>> queryList(String sql) throws SQLException {
    List<Map<String, String>> outResult = new LinkedList<>();
    Statement statement = connection.createStatement();
    ResultSet resultSet = statement.executeQuery(sql);
    while (resultSet.next()) {
      Map<String, String> map = new HashMap<>();
      for (int i = 1;i <= resultSet.getMetaData().getColumnCount();i++) {
        map.put(resultSet.getMetaData().getColumnLabel(i), resultSet.getString(i));
      }
      outResult.add(map);
    }
    statement.close();
    return outResult;
  }

  /**
   * 查詢單行輸出Map<String, String>
   * */
  public Map<String, String> queryMap(String sql) throws SQLException {
    return queryList(sql).get(0);
  }

  public void closeConnection() throws SQLException {
    connection.close();
  }

}

3、基礎類型數據小配

package com.xxxx.xxxx.tool.execl.bean;

import lombok.Getter;

/**
 * 合併命令類型
 * */
@Getter
public enum CommandRuleEnum {
  MERGE_BY_KEY("[0-9a-zA-Z]+[=][0-9a-zA-Z]+[,][0-9a-zA-Z]+",  1, "通過key匹配合並數據"),
  MERGE("([0-9a-zA-Z]+[,])+[0-9a-zA-Z]+", 2, "直接合並數據");

  private String rule;

  private Integer operation;

  private String remark;

  CommandRuleEnum(String rule, Integer operation, String remark) {
    this.rule = rule;
    this.operation = operation;
    this.remark = remark;
  }
}


package com.xxxx.xxxx.tool.execl.bean;

import lombok.Getter;

/**
 * 命令類型
 * */
@Getter
public enum CommandTypeEnum {

  SQL("sql:", "sql腳本"),
  MERGE("merge:", "合併導出");

  private String type;

  private String remark;

  CommandTypeEnum(String type, String remark) {
    this.type = type;
    this.remark = remark;
  }

}

package com.xxxx.xxxx.tool.execl.bean;

import lombok.Getter;

/**
 * sql輸出數據類型
 * */
@Getter
public enum DataType {
  LIST("list"),
  MAP("map");

  private String type;

  DataType(String type) {
    this.type = type;
  }
}


package com.xxxx.xxxx.tool.execl.bean;

import com.alibaba.excel.annotation.ExcelProperty;
import lombok.Getter;
import lombok.Setter;

@Getter
@Setter
public class InputRowModel {

  @ExcelProperty(value = "腳本", index = 0)
  private String command;

  @ExcelProperty(value = "類型", index = 1)
  private String type;

  @ExcelProperty(value = "別名", index = 2)
  private String alias;

  @ExcelProperty(value = "描述", index = 3)
  private String remark;

}

4、一個模板配置，一個腳本配置

腳本

5、xml的配置文件

<project xmlns="http://maven.apache.org/POM/4.0.0"
	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<parent>
		<groupId>com.xxx.xx</groupId>
		<artifactId>xxxx-tools</artifactId>
		<version>1.0.0.0</version>
	</parent>
	<artifactId>xxxx-tools-xlsxReportEngine</artifactId>

	<dependencies>
		<dependency>
			<groupId>com.alibaba</groupId>
			<artifactId>easyexcel</artifactId>
			<version>2.2.4</version>
		</dependency>
		<dependency>
			<groupId>mysql</groupId>
			<artifactId>mysql-connector-java</artifactId>
			<version>8.0.13</version>
		</dependency>
		<dependency>
			<groupId>org.projectlombok</groupId>
			<artifactId>lombok</artifactId>
			<version>1.18.12</version>
			<scope>provided</scope>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-web</artifactId>
		</dependency>
		<dependency>
			<groupId>com.alibaba</groupId>
			<artifactId>druid-spring-boot-starter</artifactId>
		</dependency>
		<!-- 郵件 -->
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-mail</artifactId>
			<version>2.2.6.RELEASE</version>
		</dependency>
		<dependency>
			<groupId>com.github.pagehelper</groupId>
			<artifactId>pagehelper-spring-boot-starter</artifactId>
			<exclusions>
				<exclusion>
					<groupId>org.springframework.boot</groupId>
					<artifactId>spring-boot-starter</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
	</dependencies>

	<build>
		<plugins>
			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-compiler-plugin</artifactId>
				<version>3.6.0</version>
				<configuration>
					<source>1.8</source>
					<target>1.8</target>
				</configuration>
			</plugin>
			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-shade-plugin</artifactId>
				<executions>
					<execution>
						<phase>package</phase>
						<goals>
							<goal>shade</goal>
						</goals>
						<configuration>
							<transformers>
								<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
									<mainClass>com.aspire.execlreporttool.ExcelReportToolAppliaction</mainClass>
								</transformer>
							</transformers>
						</configuration>
					</execution>
				</executions>
			</plugin>
		</plugins>
	</build>

</project>