spark遍歷hdfs目錄下所有文件

 

1、查看hdfs   /home/data/test  目錄下的所有文件

2、使用 org.apache.hadoop.fs.FileSystem 類遍歷hdfs文件

package com.xtd.hdfs

import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, FileUtil, Path}
import scala.collection.mutable.{ArrayBuffer, ListBuffer}

object HDFSUtils {

  def main(args: Array[String]): Unit = {
    val files = getHDFSFiles("hdfs://ns1/home/data/test")
    files.foreach(println(_))
  }

  /**
   * 給定hdfs目錄返回目錄下文件名的集合
   * @param hdfsDirectory
   * @return Array[String]
   */
  def getHDFSFiles(hdfsDirectory:String): Array[String] ={
    val configuration:Configuration = new Configuration()
//    configuration.set("fs.defaultFS", hdfsFileName)
    val fileSystem:FileSystem = FileSystem.get(configuration)
    val fsPath: Path = new Path(hdfsDirectory)
    val iterator = fileSystem.listFiles(fsPath, true)
    val list = new ListBuffer[String]
    while (iterator.hasNext) {
      val pathStatus = iterator.next()
      val hdfsPath = pathStatus.getPath
      val fileName = hdfsPath.getName
      list += fileName // list.append(fileName)
    }
    fileSystem.close()
    list.toArray
  }

}

 3、運行效果