C# Hadoop学习笔记（一）—环境安装

一、安装环境

1，前期准备：官网下载“NuGet Package Manager”，按自己已有的VS环境下载对应版本；

2，利用NuGet下载Hadoop For .NET SDK,地址“http://hadoopsdk.codeplex.com/”

3，安装。

4，通过HDInsight，安装Windows Azure，目前是预览版本。

5，参照网址“http://blogs.msdn.com/b/data_otaku/archive/2013/08/14/hadoop-for-net-developers.aspx” 系统学习API

二、测试DEMO

[csharp] view plain copy

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

using System.Threading.Tasks;

using Microsoft.Hadoop;

using Microsoft.Hadoop.MapReduce;

using Microsoft.Hadoop.WebClient.WebHCatClient;

using System.Diagnostics;

using System.IO;

using System.IO.MemoryMappedFiles;

namespace HadoopConsol

{

class Program

{

static void Main(string[] args)

{

Stopwatch sw = new Stopwatch();

long hadoopTime=0;

long normalTime=0;

sw.Start();

//start hadoop

Console.WriteLine(" Hadoop Process Strating ....");

#region Hadoop time

#region hadoopconnet

Console.WriteLine(" Hadoop Connect Strating ....");

//establish job configuration

HadoopJobConfiguration myConfig = new HadoopJobConfiguration();

myConfig.InputPath = "/demo/simple/in";

myConfig.OutputFolder = "/demo/simple/out";

//connect to cluster

Uri myUri = new Uri("http://localhost");

string userName = "hadoop";

string passWord = null;

IHadoop myCluster = Hadoop.Connect(myUri, userName, passWord);

hadoopTime += sw.ElapsedMilliseconds;

Console.WriteLine(" Hadoop Connect End.");

Console.WriteLine(" Hadoop Connect time:" + sw.ElapsedMilliseconds);

#endregion

#region hadoopmapreduce

sw.Reset();

sw.Start();

Console.WriteLine(" Hadoop MapReduce Strating ....");

//execute mapreduce job

MapReduceResult jobResult =

myCluster.MapReduceJob.Execute<MySimpleMapper, MySimpleReducer>(myConfig);

hadoopTime += sw.ElapsedMilliseconds;

Console.WriteLine(" Hadoop MapReduce End.");

Console.WriteLine(" Hadoop MapReduce Time:"+sw.ElapsedMilliseconds);

#endregion

#region Hadoop End

sw.Reset();

sw.Start();

Console.WriteLine(" Hadoop Endprocess Strating ....");

//write job result to console

int exitCode = jobResult.Info.ExitCode;

string exitStatus = "Failure";

if (exitCode == 0) exitStatus = "Success";

exitStatus = exitCode + " (" + exitStatus + ")";

Console.WriteLine();

Console.Write("Exit Code = " + exitStatus);

Console.WriteLine(" Hadoop Endprocess End.");

hadoopTime += sw.ElapsedMilliseconds;

Console.WriteLine(" Hadoop Exit Time:" + sw.ElapsedMilliseconds);

Console.WriteLine(" Hadoop Process All Time:" + hadoopTime);

#endregion

#endregion

#region Normal time

//start Normal

Console.WriteLine(" Normal Process Strating ....");

sw.Reset();

sw.Start();

//normal process

#region Normal Process

int myevenCount = 0;

int myeventSum = 0;

int myoddCount = 0;

int myoddSum = 0;

StreamReader fs = new StreamReader(@"c:\TEMP\integers.txt");

while (fs.Peek() >= 0)

{

string strTemp = fs.ReadLine();

if (Int32.Parse(strTemp) % 2 == 0)

{

myevenCount++;

myeventSum += Int32.Parse(strTemp);

}

else

{

myoddCount++;

myoddSum += Int32.Parse(strTemp);

}

}

//MemoryMappedFile m = MemoryMappedFile.

Console.WriteLine("even:" + "\t" + myevenCount + "\t" + myeventSum);

Console.WriteLine("odd:" + "\t" + myoddCount + "\t" + myoddSum);

#endregion

Console.WriteLine(" Normal Process End.");

normalTime += sw.ElapsedMilliseconds;

Console.WriteLine(" Normal Exit Time:" + sw.ElapsedMilliseconds);

Console.WriteLine(" Normal Process All Time:" + normalTime);

#endregion

sw.Stop();

Console.Read();

}

}

public class MySimpleMapper : MapperBase

{

public override void Map(string inputLine, MapperContext context)

{

//interpret the incoming line as an integer value

int value = int.Parse(inputLine);

//determine whether value is even or odd

string key = (value % 2 == 0) ? "even" : "odd";

//output key assignment with value

context.EmitKeyValue(key, value.ToString());

}

}

public class MySimpleReducer : ReducerCombinerBase

{

public override void Reduce(

string key, IEnumerable<string> values, ReducerCombinerContext context

)

{

//initialize counters

int myCount = 0;

int mySum = 0;

//count and sum incoming values

foreach (string value in values)

{

mySum += int.Parse(value);

myCount++;

}

//output results

context.EmitKeyValue(key, myCount + "\t" + mySum);

}

}

}

三、测试结果