提交Spark第一个程序

第一个程序是计算session 日志查询排行榜

SougouQA

package week2

/**
* Created by root on 15-8-21.
*/

import org.apache.spark.{SparkContext, SparkConf}
import org.apache.spark.SparkContext._

object SougouQA{
def main(args: Array[String]) {
if (args.length == 0) {
System.err.println("Usage: SougouQA <file1> <file1>")
System.exit(1)
}

val conf = new SparkConf().setAppName("SougouQA")
val sc = new SparkContext(conf)
sc.textFile(args(0)).map(_.split("\t")).filter(_.length==6).map(x=>(x(1),1)).reduceByKey(_+_).map(x=>(x._2,x._1)).sortByKey(false).map(x=>(x._2,x._1)).saveAsTextFile(args(1))

sc.stop()
}
}

join

package week2

/**
* Created by root on 15-8-21.
*/
import org.apache.spark.{SparkContext, SparkConf}
import org.apache.spark.SparkContext._

object join {
def main(args: Array[String]) {
if (args.length == 0) {
System.err.println("Usage: join <file1> <file2>")
System.exit(1)
}

val conf = new SparkConf().setAppName("join")
val sc = new SparkContext(conf)
val format = new java.text.SimpleDateFormat("yyyy-MM-dd")
case class Register (d: java.util.Date, uuid: String, cust_id: String, lat: Float,lng: Float)
case class Click (d: java.util.Date, uuid: String, landing_page: Int)
val reg = sc.textFile(args(0)).map(_.split("\t")).map(r => (r(1), Register(format.parse(r(0)), r(1), r(2), r(3).toFloat, r(4).toFloat)))
val clk = sc.textFile(args(1)).map(_.split("\t")).map(c => (c(1), Click(format.parse(c(0)), c(1), c(2).trim.toInt)))
reg.join(clk).take(2).foreach(println)

sc.stop()
}
}

SougouQA的运行

bin/spark-submit –master spark://moon:7077 –class week2.SougouQA week2.jar hdfs://localhost:9000/user/SogouQ1.txt hdfs://localhost:9000/user/week2output

合并输出结果

hdfs dfs -getmerge hdfs://localhost:9000/user/week2output  result1

这个resul1是在当前目录下，可以看看访问前10名

head result1

(b3c94c37fb154d46c30a360c7941ff7e,676)

(cc7063efc64510c20bcdd604e12a3b26,613)

(955c6390c02797b3558ba223b8201915,391)

(b1e371de5729cdda9270b7ad09484c4f,337)

(6056710d9eafa569ddc800fe24643051,277)

(637b29b47fed3853e117aa7009a4b621,266)

(c9f4ff7790d0615f6f66b410673e3124,231)

(dca9034de17f6c34cfd56db13ce39f1c,226)

(82e53ddb484e632437039048c5901608,221)

(c72ce1164bcd263ba1f69292abdfdf7c,214)

join的运行

bin/spark-submit –master spark://moon:7077 –class week2.join week2.jar hdfs://localhost:9000/user/join/reg.tsv hdfs://localhost:9000/user/join/clk.tsv

输出结果 ##