菜鳥使用spark總結

1、RDD類型的使用foreach 是不能在裏面給任何一個集合賦值

2、RDD類型轉換爲collect()之後再foreach是可以賦值

3、RDD可以轉換map返

private def saveSampleData(thunnelSampleEsIndexPrefi: String, thunnelSampleEsIndexType: String, currentDay: String, result_df: RDD[(String, Iterable[Row])], sc: SparkContext, thunnelThreshold: Long, domainListByMainDomain:Broadcast[Map[String, Iterable[String]]]): Unit = {
    result_df.map(x => {
      val srcIp = x._1
      x._2.toList.foreach(row =>{
        if (row.getLong(2) > thunnelThreshold){
          val mainDomain = row.getString(0)
          //domain_src_ip
          var key = row.getString(0).toString + "_" + row.getString(1).toString
          //用於保存樣本詳細信息
          val domainList:List[String] = domainListByMainDomain.value.get(key).get.toList
          domainList.foreach(x=>{
            var json: JSONObject = new JSONObject()
            json.put("id", UUID.randomUUID().toString)
            json
          })
        }
      })
    }).saveToEs(path)
    
  }

回一個json存儲到es