解析氣象Grib文件實例

GRIB 碼是與計算機無關的壓縮的二進制編碼,主要用來表示數值天氣預報的產品資料。現行的GRIB 碼版本有GRIB1 和GRIB2 兩種格式。 GRIB2較之GRIB1具有加大優點而被廣泛使用。如：表示多維數據、模塊性結構、支持多種壓縮方式、IEEE標準浮點表示法等。
首先需要引入的三方庫

<repositories>  
        <repository>  
            <id>unidata</id>  
            <name>THREDDS</name>  
            <url>https://artifacts.unidata.ucar.edu/content/repositories/unidata-releases/</url>  
</repositories>  
  
  
   <dependency>     
            <groupId>joda-time</groupId>  
            <artifactId>joda-time</artifactId>  
            <version>2.3</version>  
        </dependency>  
        <dependency>  
            <groupId>org.glassfish</groupId>  
            <artifactId>javax.json</artifactId>  
            <version>1.0.3</version>  
        </dependency>  
        <dependency>  
            <groupId>com.lexicalscope.jewelcli</groupId>  
            <artifactId>jewelcli</artifactId>  
            <version>0.8.8</version>  
        </dependency>  
        <dependency>  
            <groupId>edu.ucar</groupId>  
            <artifactId>grib</artifactId>  
            <version>4.3.19</version>  
</dependency>

通過讀取grib 文件來解析數據

public void fileData() {  
        try{  
//grib 文件夾裏包含N個grib 文件  
            File folder = new File(environment.getProperty("weatherFolder"));  
            File[] files = folder.listFiles();  
            for (File file : files) {  
                String[] args = {"--data","--names",file.getPath()};  
                verify(args);  
            }  
              
        }catch(Exception e){  
            log.error("文件導入發生異常:{}",e);  
        }  
          
    }

public  void verify(String[] args) {  
        Options options = CliFactory.parseArguments(Options.class, args);  
        try {  
            if (!options.getFile().exists()) {  
                log.error("Cannot find input file {0}",options.getFile());  
                return;  
            }  
            if(options.getFile().isDirectory()){  
                return;  
            }  
            log.info("讀取  {} 氣象文件",options.getFile().getName());  
            new Grib2Json(options.getFile(), options).read(table);  
      
        }catch (Exception e) {  
            log.error("verify error:{}",e);  
            e.printStackTrace();  
            System.exit(1);  
        }

package com.kedalo.databus.grib2;  
  
import com.lexicalscope.jewel.cli.*;  
  
import java.io.File;  
  
  
/** 
 * @date 2019-09-17 
 * 
 * 
 * @author liyulong 
 * 該功能實現一部分 後續可擴展 
 */  
@CommandLineInterface(application="grib2json", order=OptionOrder.LONGNAME)  
public interface Options {  
  
    @Option(longName="help", shortName="h", description="幫助命令")  
    boolean getShowHelp();  
  
    @Option(longName="names", shortName="n", description="打印返回字段名稱")  
    boolean getPrintNames();  
  
    @Option(longName="data", shortName="d", description="打印數據字段")  
    boolean getPrintData();  
  
    @Option(longName="compact", shortName="c", description="轉化爲JSON格式")  
    boolean isCompactFormat();  
  
    @Option(longName="verbose", shortName="v", description="啓動日誌記錄")  
    boolean getEnableLogging();  
  
    @Option(  
        longName="output",  
        shortName="o",  
        description="輸出命令  例:-o 文件路徑",  
        defaultToNull=true)  
    File getOutput();  
  
    @Unparsed(name="FILE", defaultToNull=true)  
    File getFile();  
    @Option(  
        longName={"filter.discipline", "fd"},  
        description="行業過濾起 暫時無用",  
        defaultToNull=true)  
    Integer getFilterDiscipline();  
  
    @Option(  
        longName={"filter.category", "fc"},  
        description="類型過濾器 暫時無用",  
        defaultToNull=true)  
    Integer getFilterCategory();  
  
    @Option(  
        longName={"filter.parameter", "fp"},  
        description="參數過濾器  暫時無用",  
        defaultToNull=true)  
    String getFilterParameter();  
  
    @Option(  
        longName={"filter.surface", "fs"},  
        description="surface 字段暫時無用",  
        defaultToNull=true)  
    Integer getFilterSurface();  
  
    @Option(  
        longName={"filter.value", "fv"},  
        description="過濾 surface 內容 暫時無用",  
        defaultToNull=true)  
    Double getFilterValue();  
     
}

import javax.json.JsonNumber;  
import java.math.BigDecimal;  
import java.math.BigInteger;  
  
  
/** 
 * 2014-01-17 
 * @author liyulong 
 */  
final class FloatValue implements JsonNumber {  
  
    private final float value;  
    private BigDecimal bd;  
  
    FloatValue(float value) {  
        this.value = value;  
    }  
  
    @Override   
    public ValueType getValueType() {  
        return ValueType.NUMBER;  
    }  
  
    @Override   
    public String toString() {  
        if (Float.isNaN(value)) {  
            return "\"NaN\"";  
        }  
        else if (value == Float.POSITIVE_INFINITY) {  
            return "\"-Infinity\"";  
        }  
        else if (value == Float.NEGATIVE_INFINITY) {  
            return "\"Infinity\"";  
        }  
        else {  
            return Float.toString(value);  
        }  
    }  
  
    @Override   
    public boolean isIntegral() {  
        return bigDecimalValue().scale() == 0;  
    }  
  
    @Override   
    public int intValue() {  
        return (int)value;  
    }  
  
    @Override   
    public int intValueExact() {  
        return bigDecimalValue().intValueExact();  
    }  
  
    @Override   
    public long longValue() {  
        return (long)value;  
    }  
  
    @Override   
    public long longValueExact() {  
        return bigDecimalValue().longValueExact();  
    }  
  
    @Override   
    public BigInteger bigIntegerValue() {  
        return bigDecimalValue().toBigInteger();  
    }  
  
    @Override   
    public BigInteger bigIntegerValueExact() {  
        return bigDecimalValue().toBigIntegerExact();  
    }  
  
    @Override   
    public double doubleValue() {  
        return (double)value;  
    }  
  
    @Override   
    public BigDecimal bigDecimalValue() {  
        return bd != null ? bd : (bd = new BigDecimal(value));  
    }  
  
    @Override   
    public boolean equals(Object that) {  
        return that instanceof JsonNumber && this.bigDecimalValue().equals(((JsonNumber)that).bigDecimalValue());  
    }  
  
    @Override   
    public int hashCode() {  
        return bigDecimalValue().hashCode();  
    }  
}

 
import static ucar.grib.GribNumbers.BIT_5;  
import static ucar.grib.GribNumbers.UNDEFINED;  
import static ucar.grib.GribNumbers.isBitSet;  
import static ucar.grib.grib2.Grib2Tables.codeTable3_1;  
import static ucar.grib.grib2.Grib2Tables.codeTable3_2;  
import static ucar.grib.grib2.Grib2Tables.codeTable4_0;  
import static ucar.grib.grib2.Grib2Tables.codeTable4_3;  
import static ucar.grib.grib2.Grib2Tables.codeTable4_5;  
import static ucar.grib.grib2.ParameterTable.getCategoryName;  
import static ucar.grib.grib2.ParameterTable.getParameterName;  
import static ucar.grib.grib2.ParameterTable.getParameterUnit;  
  
import java.io.IOException;  
import java.util.ArrayList;  
import java.util.Date;  
import java.util.HashMap;  
import java.util.List;  
import java.util.Map;  
import org.joda.time.DateTime;  
import org.joda.time.DateTimeZone;  
  
import com.kedalo.databus.bean.GribWeather;  
import com.kedalo.databus.util.WeatherUtil;  
  
import ucar.grib.grib2.Grib2Data;  
import ucar.grib.grib2.Grib2GDSVariables;  
import ucar.grib.grib2.Grib2IdentificationSection;  
import ucar.grib.grib2.Grib2IndicatorSection;  
import ucar.grib.grib2.Grib2Pds;  
import ucar.grib.grib2.Grib2Record;  
/** 
 * @author liyulong 
 * @date 2019-09-17 
 * grib 保存 
 * */  
public class GribRecord {  
      
      private final Grib2Record record;  
      private final Grib2IndicatorSection ins;  
      private final Options options;  
      private final Grib2IdentificationSection ids;  
      private final Grib2Pds pds;  
      private final Grib2GDSVariables gds;  
        
       
        
      GribRecord(Grib2Record record, Options options) {  
            this.record = record;  
            this.options = options;  
            this.ins = record.getIs();  
            this.ids = record.getId();  
            this.pds = record.getPDS().getPdsVars();  
            this.gds = record.getGDS().getGdsVars();  
     }  
        
       
     public Map<String,Object> getHead(){  
         int productDef = pds.getProductDefinitionTemplate();  
         int discipline = ins.getDiscipline();  
         int paramCategory = pds.getParameterCategory();  
         int paramNumber = pds.getParameterNumber();  
         int gridTemplate = gds.getGdtn();  
         Map<String,Object> recordData = new HashMap<String,Object>();  
        //學科  
         recordData.put("discipline", ins.getDiscipline());  
         recordData.put("disciplineName", ins.getDisciplineName());  
         //Grib  
         recordData.put("gribEdition", ins.getGribEdition());  
            //中心點 固定值38  
         recordData.put("center", ids.getCenter_id());  
         recordData.put("centerName",WeatherUtil.codeToStr(ids.getCenter_id()));  
            //子中心點  
         recordData.put("subcenter", ids.getSubcenter_id());  
           
         recordData.put("timeLong", ids.getRefTime());  
         recordData.put("refTime", new Date(ids.getRefTime()));   
//       recordData.put("refTime", new DateTime(ids.getRefTime()).toDate());  
         recordData.put("time", WeatherUtil.addHourTime(new DateTime(ids.getRefTime()).toDate(), pds.getForecastTime()));  
         recordData.put("significanceOfRT", ids.getSignificanceOfRT());  
         recordData.put("significanceOfRTName",  ids.getSignificanceOfRTName());  
         recordData.put("productStatus", ids.getProductStatus());  
         recordData.put("productStatusName", ids.getProductStatusName());  
         recordData.put("productType", ids.getProductType());  
         recordData.put("productTypeName", ids.getProductStatusName());  
         recordData.put("productDefinitionTemplate", productDef);  
         recordData.put("productDefinitionTemplateName", codeTable4_0(productDef));  
         recordData.put("parameterCategory", paramCategory);  
         recordData.put("parameterCategoryName",getCategoryName(discipline, paramCategory));  
         recordData.put("parameterNumber", paramNumber);  
         recordData.put("parameterNumberName",getParameterName(discipline, paramCategory, paramNumber));  
         recordData.put("parameterUnit", getParameterUnit(discipline, paramCategory, paramNumber));  
         recordData.put("genProcessType", pds.getGenProcessType());  
         recordData.put("genProcessTypeName", codeTable4_3(pds.getGenProcessType()));  
         recordData.put("forecastTime", pds.getForecastTime());  
         recordData.put("surface1Type", pds.getLevelType1());  
         recordData.put("surface1TypeName", codeTable4_5(pds.getLevelType1()));  
         recordData.put("surface1Value", pds.getLevelValue1());  
         recordData.put("surface2Type", pds.getLevelType2());  
         recordData.put("surface2TypeName",codeTable4_5(pds.getLevelType2()));  
         recordData.put("surface2Value", pds.getLevelValue2());  
         recordData.put("gridDefinitionTemplate", gridTemplate);  
         recordData.put("gridDefinitionTemplateName",codeTable3_1(gridTemplate));  
         recordData.put("numberPoints", gds.getNumberPoints());  
         switch (gridTemplate) {  
                case 0:  // Template 3.0  
                case 1:  // Template 3.1  
                case 2:  // Template 3.2  
                case 3:  // Template 3.3  
                    writeLonLatGrid(recordData);  
                break;  
                case 10:  // Template 3.10  
                    writeMercatorGrid(recordData);  
                    break;  
                case 20:  // Template 3.20  
                    writePolarStereographicGrid(recordData);  
                    break;  
                case 30:  // Template 3.30  
                    writeLambertConformalGrid(recordData);  
                    break;  
                case 40:  // Template 3.40  
                case 41:  // Template 3.41  
                case 42:  // Template 3.42  
                case 43:  // Template 3.43  
                    writeLonLatGrid(recordData);  
                    break;  
                case 90:  // Template 3.90  
                    writeSpaceOrOrthographicGrid(recordData);  
                    break;  
                case 204:  // Template 3.204  
                    writeCurvilinearGrid(recordData);  
                    break;  
            }  
              
         return recordData;  
          
    }  
     private void writeCurvilinearGrid(Map<String,Object> recordData) {  
            writeGridShape(recordData);  
            writeGridSize(recordData);  
    }  
       
     private void writeSpaceOrOrthographicGrid(Map<String,Object> recordData) {  
            writeGridShape(recordData);  
            writeGridSize(recordData);  
            writeAngle(recordData);  
            writeLonLatBounds(recordData);  
            recordData.put("lop", gds.getLop());   
            recordData.put("lap", gds.getLap());   
            recordData.put("xp", gds.getXp());      
            recordData.put("yp", gds.getYp());     
            recordData.put("nr", gds.getNr());     
            recordData.put("xo", gds.getXo());     
            recordData.put("yo", gds.getYo());     
     }  
       
     private void writeGridShape(Map<String,Object> recordData) {  
          recordData.put("shape",codeTable3_2(gds.getShape()));  
          recordData.put("shapeName", codeTable3_2(gds.getShape()));  
            switch (gds.getShape()) {  
                case 1:  
                     recordData.put("earthRadius", gds.getEarthRadius());  
                    break;  
                case 3:   
                     recordData.put("majorAxis", gds.getMajorAxis());  
                     recordData.put("minorAxis", gds.getMinorAxis());  
                    break;  
            }  
     }  
       
     private void writeGridSize(Map<String,Object> recordData) {  
         recordData.put("gridUnits", gds.getGridUnits());  
         recordData.put("resolution", gds.getResolution());  
         recordData.put("winds", isBitSet(gds.getResolution(), BIT_5) ? "relative" : "true");  
         recordData.put("scanMode", gds.getScanMode());  
         recordData.put("nx", gds.getNx());    
         recordData.put("ny", gds.getNy());    
    }  
      
    private void writeAngle(Map<String,Object> recordData) {  
        recordData.put("angle", gds.getAngle());  
        recordData.put("basicAngle", gds.getBasicAngle());  
        recordData.put("subDivisions", gds.getSubDivisions());  
    }  
       
      
        
      private void putIfSet(Map<String,Object> recordData,String key,float value){  
            if ( value != UNDEFINED) {  
                recordData.put(key, value);  
            }  
      }  
        
      private void writeLonLatBounds(Map<String,Object> recordData) {  
          putIfSet(recordData,"lo1",gds.getLo1());  
          putIfSet(recordData,"la1",gds.getLa1());  
          putIfSet(recordData,"lo2",gds.getLo2());  
          putIfSet(recordData,"la2",gds.getLa2());  
          putIfSet(recordData,"dx",gds.getDx());  
          putIfSet(recordData,"dy",gds.getDy());  
              
        }  
        
      private void writeRotationAndStretch(Map<String,Object> recordData) {  
          putIfSet(recordData,"spLon", gds.getSpLon());   
          putIfSet(recordData,"spLat", gds.getSpLat());  
          putIfSet(recordData,"rotationAngle", gds.getRotationAngle());  
          putIfSet(recordData,"poleLon", gds.getPoleLon());   
          putIfSet(recordData,"poleLat", gds.getPoleLat());   
          putIfSet(recordData,"stretchingFactor", gds.getStretchingFactor());  
     }  
        
     private void writeLonLatGrid(Map<String,Object> recordData) {  
         writeGridShape(recordData);  
         writeGridSize(recordData);  
         writeAngle(recordData);  
         writeLonLatBounds(recordData);  
         writeRotationAndStretch(recordData);  
         putIfSet(recordData,"np", gds.getNp());    
     }  
        
     private void writeMercatorGrid(Map<String,Object> recordData) {  
         writeGridShape(recordData);  
         writeGridSize(recordData);  
         writeAngle(recordData);  
         writeLonLatBounds(recordData);  
    }  
     private void writePolarStereographicGrid(Map<String,Object> recordData) {  
         writeGridShape(recordData);  
         writeGridSize(recordData);  
         writeLonLatBounds(recordData);  
     }  
     private void writeLambertConformalGrid(Map<String,Object> recordData) {  
         writeGridShape(recordData);  
         writeGridSize(recordData);  
         writeLonLatBounds(recordData);  
         writeRotationAndStretch(recordData);  
         recordData.put("laD", gds.getLaD());  
         recordData.put("loV", gds.getLoV());  
         recordData.put("projectionFlag", gds.getProjectionFlag());  
         recordData.put("latin1", gds.getLatin1());   
         recordData.put("latin2", gds.getLatin2());    
    }  
       
     List<FloatValue> readData(Grib2Data gd) throws IOException {  
         List<FloatValue> list = new ArrayList<FloatValue>();  
         float[] data = gd.getData(record.getGdsOffset(), record.getPdsOffset(), ids.getRefTime());  
         if (data != null) {  
             for (float value : data) {  
                 list.add(new FloatValue(value));  
             }  
         }  
         return list;  
    }  
      
}

Grib2Json

因爲grib 數據格式是這樣的以時間的方式保存一個格點所有數據

而 mongoDB 的保存方式以座標爲主的保存方式所以在業務邏輯上想找座標對應所有時間的數據然後依次座標遍歷所有當前座標的時間數據

import org.bson.Document;  
import org.slf4j.Logger;  
import org.slf4j.LoggerFactory;  
  
import com.kedalo.databus.comm.MongoClientCommon;  
import com.kedalo.databus.content.SpringContext;  
import com.kedalo.databus.util.WeatherUtil;  
import ucar.grib.grib2.*;  
import ucar.unidata.io.RandomAccessFile;  
import java.io.*;  
import java.util.*;  
  
  
/** 
 * @author liyulong 
 * Date 2019-09-17 
 * 解析Grib文件 
 */  
public final class Grib2Json{  
  
    private static final Logger log = LoggerFactory.getLogger(Grib2Json.class);  
  
    private final File file;  
    private final Options option;  
      
   MongoClientCommon mongoClient = (MongoClientCommon) SpringContext.getBean(MongoClientCommon.class);  
  
    public Grib2Json(File file, Options option) {  
        if (!file.exists()) {  
            log.error("Cannot find input file {0}",file);  
            throw new IllegalArgumentException("Cannot find input file: " + file);  
        }  
        this.file = file;  
        this.option = option;  
    }  
  
    
/***  這裏是處理業務方法  這裏是將數據解析出來存入MongoDB 因爲業務原因 grib 保存結構和業務平臺需要的結構相反 一個是扁平是存儲 一個是 樹形存儲 所以 業務邏輯有點繞，請自行忽略  */  
    private void read(RandomAccessFile raf, Grib2Input input, Options options,String table) throws IOException {  
           List<Grib2Record> records = input.getRecords();  
        float startLon = 0;//開始經度  
        float startLat = 0;//開始緯度  
        float x = 0;//緯度每次變化的值  
        float y = 0;//經度每次變化的值  
        float endLon=0;//結束的經度  
        float endLat=0;//結束的緯度  
        float[] data = null;//獲取data個數  
        int forecast = 0;  
        if(records.size() > 0){  
            Grib2Record temp = records.get(0);  
            GribRecord rw = new GribRecord(temp, options);  
            Map<String,Object> map = rw.getHead();  
            forecast = (int)map.get("forecastTime");  
            startLon = WeatherUtil.toFloat((float)map.get("lo1"));  
            endLon = WeatherUtil.toFloat((float) map.get("lo2"));  
            x = WeatherUtil.toFloat((float) map.get("dx"));  
            y = WeatherUtil.toFloat((float) map.get("dy"));  
            startLat = WeatherUtil.toFloat((float) map.get("la1"));  
            endLat = WeatherUtil.toFloat((float) map.get("la2"));  
            data = new Grib2Data(raf).getData(temp.getGdsOffset(), temp.getPdsOffset(),  temp.getId().getRefTime());  
        }  
        int lonCount = WeatherUtil.scale(WeatherUtil.sub(endLon, startLon)/y, 0).intValue();  
        int latCount = WeatherUtil.scale(WeatherUtil.sub(endLat, startLat)/x, 0).intValue();  
        float tempLon = 0;  
        float tempLat = 0;  
        int count = 0;  
        for (int i = 0; i < data.length; i++) {  
            tempLon = startLon + (y*(i%(lonCount+1)));  
            tempLat = startLat +(x*count%latCount);  
            count = i/(lonCount+1);  
            Document doc =  new Document();  
              
            Map<String, Object> documentMap = new HashMap<String, Object>();   
            documentMap.put("type", "Point");  
            List<Float> list = new ArrayList<Float>();  
            list.add(tempLon);  
            list.add(tempLat);  
            documentMap.put("coordinates",list);  
            doc.append("loc",documentMap);  
              
            list = new ArrayList<Float>();  
            for (int j = 0; j < records.size(); j++) {  
                Grib2Record temp = records.get(j);  
                GribRecord rw = new GribRecord(temp, options);  
                Map<String,Object> map = rw.getHead();  
                doc.append("forecastTime",forecast)  
                .append("parameterNumber", map.get("parameterNumber"))  
                .append("parameterNumberName",  map.get("parameterNumberName"))  
                .append("parameterCategory", map.get("parameterCategory"))  
                .append("parameterCategoryName",  map.get("parameterCategoryName"))  
                .append("parameterUnit",  map.get("parameterUnit"))  
                .append("refTime", map.get("refTime"));  
                data = new Grib2Data(raf).getData(temp.getGdsOffset(), temp.getPdsOffset(),  temp.getId().getRefTime());  
                list.add(data[i]);  
            }  
            doc.append("data", list);  
            //保存至MongoDB  
            mongoClient.insertOne(table,doc);  
        }  
        mongoClient.createIndex(table, "loc", "2dsphere");  
    }  
  
   
    public void read(String table) throws IOException {  
        RandomAccessFile raf = new RandomAccessFile(file.getPath(), "r");  
        raf.order(RandomAccessFile.BIG_ENDIAN);  
        Grib2Input input = new Grib2Input(raf);  
        input.scan(false, false);  
        read(raf, input, option,table);  
    }  
}

解析氣象Grib文件實例

Python實現大麥網搶票的四大關鍵技術點解析

salesforce零基礎學習（一百三十八）零碎知識點小總結（十）

KubeEdge v1.17.0發佈！數據處理能力與易用性全面提升

一文教你基於LangChain和ChatGLM3搭建本地知識庫問答

webservice ssl 2 keyStore和truststore區別

Python實現大麥網搶票的四大關鍵技術點解析

MySQL 核心模塊揭祕 | 17 期 | InnoDB 有哪幾種行鎖？

https://yachay.unat.edu.pe/blog/index.php?comment_area=format_blog&comment_component=blog&comment_co

linux以太網驅動總結