载入中。。。 'S bLog
 
载入中。。。
 
载入中。。。
载入中。。。
载入中。。。
载入中。。。
载入中。。。
 
填写您的邮件地址,订阅我们的精彩内容:


 
hadoop权威指南第二章示例
[ 2011/6/9 20:31:00 | By: 梦翔儿 ]
 

hadoop权威指南第二章示例

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class MaxTemperature {
 static class MaxTemperatureMapper extends Mapper<LongWritable, Text, Text, IntWritable>{
private  static final int MISSING = 9999;

public void map(LongWritable key, Text value,
  Context context)
  throws IOException, InterruptedException {
 // TODO Auto-generated method stub
 String line = value.toString(); //get as string
 String year = line.substring(5,9); //year
 int airTemperature = Integer.parseInt(line.substring(15,19)); //get temperature to Dec
 
 if (airTemperature != MISSING){
  context.write(new Text(year), new IntWritable(airTemperature));   
 }
}
}
 

static class MaxTemperatureReducer extends Reducer<Text, IntWritable, Text, IntWritable> {

 public void reduce(Text key, Iterable<IntWritable> values,
   Context context, Reporter reporter)
   throws IOException, InterruptedException {
  // TODO Auto-generated method stub
  System.out.println("reducer");
  int maxValue = Integer.MIN_VALUE;
  for (IntWritable value : values){
   maxValue = Math.max(maxValue, value.get());
  }
  context.write(key, new IntWritable(maxValue));
 }

}

 

 public static void main(String[] args) throws IOException {
  // TODO Auto-generated method stub
  Configuration conf = new Configuration();
  if (args.length !=2){  //judge arg number
   System.err.println("usage: MaxTemperature <input path> <output path>");
   System.exit(-1);
  }
  
  //Job job = new Job();
  Job job = new Job(conf, "NCDC");
  job.setJarByClass(MaxTemperature.class);
  
  FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));

  
  job.setMapperClass(MaxTemperatureMapper.class);
  job.setCombinerClass(MaxTemperatureReducer.class);
  job.setReducerClass(MaxTemperatureReducer.class);
  
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(IntWritable.class);
  
  try {
   System.exit(job.waitForCompletion(true) ? 0 : 1);
  } catch (InterruptedException e) {
   // TODO Auto-generated catch block
   e.printStackTrace();
  } catch (ClassNotFoundException e) {
   // TODO Auto-generated catch block
   e.printStackTrace();
  }
  
  
 }

}

=======

数据集sample.txt

aaaaa1990aaaaaa0039a
bbbbb1991bbbbbb0040a
ccccc1992cccccc0040c
ddddd1993dddddd0043d
eeeee1994eeeeee0041e
aaaaa1990aaaaaa0031a
bbbbb1991bbbbbb0020a
ccccc1992cccccc0030c
ddddd1993dddddd0033d
eeeee1994eeeeee0031e
aaaaa1990aaaaaa0041a
bbbbb1991bbbbbb0040a
ccccc1992cccccc0040c
ddddd1993dddddd0043d
eeeee1994eeeeee0041e
aaaaa1990aaaaaa0044a
bbbbb1991bbbbbb0045a
ccccc1992cccccc0041c
ddddd1993dddddd0023d
eeeee1994eeeeee0041e

============
谁能帮我看看,为什么Reduce没有执行呢?如何修正?

 
 
  • 标签:hadoop权威指南 
  •  
    Re:hadoop权威指南第二章示例
    [ 2011/6/10 0:24:24 | By: 郭(游客) ]
     
    郭(游客)for (IntWritable value : values){
    maxValue = Math.max(maxValue, value.get());
    }
    老师你把这段改成
    while (values.iterator().hasNext()) {
    maxValue = Math.max(maxValue, value.iterator().next().get());
    }
    本本快没电了,我明天试试,一直按你的思路调的,后来发现,map都不执行了,仔细分析了一下map、reduce、mapreduce分开写的程序,发现这点可疑之处,分开写的运行很好,我觉得这个循环处理的有问题,明天再做个比较
    以下为梦翔儿的回复:
    while (values.iterator().hasNext()) {
    maxValue = Math.max(maxValue, values.iterator().next().get());
    }
    这两种写法应该是一样吧,关键在于reduce过程根本就没有进去过。
     
    个人主页 | 引用 | 返回 | 删除 | 回复
     
    发表评论:
    载入中。。。

     
     
     

    梦翔儿网站 梦飞翔的地方 http://www.dreamflier.net
    中华人民共和国信息产业部TCP/IP系统 备案序号:辽ICP备09000550号

    Powered by Oblog.