将hdfs上多个文本数据生成mllib的训练集测试集

package pack
import java.io.{File, PrintWriter}
import org.apache.spark.{SparkConf, SparkContext}
import scala.sys.process._
import scala.collection.mutable.ArrayBuffer
/**
* Created by hemin on 2016/9/20.
*/
object ReadFile {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName("mktxt").setMaster("lcoal")
val sc = new SparkContext(conf)
val paths = args
getData(paths,sc)
//sark程序必要的设置
}
def getData(paths:Array[String],sc:SparkContext)= {
val index = paths.length-1   //cancer the last arg
val temp = ArrayBuffer[String]()
val writer = new PrintWriter(new File("/home/iespark/zzh/test.txt"))
val len=sc.textFile(paths(0)).filter(x=>(x!="")).collect().length
//make all data to an array
for(i<-0 to index-1){
temp ++= sc.textFile(paths(i)).filter(x=>(x!="")).collect()
}
//keep a certen length
for(i<- 0 to len-2){
for(j<- 0 to index-1){
if(j==0){
writer.write(temp(i+len*j)+"  ")
}else{
writer.write(j+"::"+temp(i+len*j)+"  ")
}
}
writer.write("\n")
}
writer.close()

val txt = sc.textFile("/home/iespark/zzh/test.txt")
txt.repartition(1).saveAsTextFile("hdfs://hadoopadmin:9000/user/iespark/zzh/test")
}
}