AnsweredAssumed Answered

Exception: wrong value class: class org.apache.hadoop.io.Text is not class org.apache.hadoop.io.LongWritable

Question asked by jtherocker on Jun 2, 2013
Latest reply on Jun 2, 2013 by jtherocker
Hi,

I'm getting this weird error (Java),  when using a combiner,

<code>
java.io.IOException: wrong value class: class org.apache.hadoop.io.Text is not class org.apache.hadoop.io.LongWritable
        at org.apache.hadoop.mapred.IFile$Writer.append(IFile.java:219)
        at org.apache.hadoop.mapred.Task$CombineOutputCollector.collect(Task.java:1179)
        at org.apache.hadoop.mapred.Task$NewCombinerRunner$OutputConverter.write(Task.java:1500)
        at org.apache.hadoop.mapreduce.TaskInputOutputContext.write(TaskInputOutputContext.java:80)
        at com.mr.test.MyDriver$MyDriverReducer.reduce(MyDriver.java:36)
        at com.mr.test.MyDriver$MyDriverReducer.reduce(MyDriver.java:1)
        at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:176)
        at org.apache.hadoop.mapred.Task$NewCombinerRunner.combine(Task.java:1521)
        at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.sortAndWrite(MapTask.java:1691)
        at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.flush(MapTask.java:1351)
        at org.apache.hadoop.mapred.MapTask$NewOutputCollector.close(MapTask.java:603)
        at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:670)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:334)
        at org.apache.hadoop.mapred.Child$4.run(Child.java:270)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:396)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1126)
        at org.apache.hadoop.mapred.Child.main(Child.java:264)

</code>
**Code:**

    public class MyDriver extends Configured implements Tool {
    
      public static class MyDriverMapper extends Mapper<LongWritable, Text, Text, LongWritable> {
             protected void map(LongWritable key, Text value,
        Context context)
        throws IOException, InterruptedException {
                   //--- some code ---
                   context.write(new Text("some_key"), new LongWritable(2131));
             }
      }
      
      public static class MyDriverReducer extends Reducer<Text, LongWritable, Text, Text> {
             protected void reduce(Text key, Iterable<LongWritable> values, Context context)
        throws IOException, InterruptedException {
                  // --- some code ---
                   context.write(key, new Text("Whatever"));
              }
      }
    
     /**
      * @param args
      * @throws Exception
      */
     public static void main(String[] args) throws Exception {
      int exitCode = ToolRunner.run(new MyDriver(),
        args);
      System.exit(exitCode);
    
     }
    
     @Override
     public int run(String[] args) throws Exception {
      Configuration conf = getConf();
      Job job = new Job(conf, "My Sample Job");
      job.setJarByClass(MyDriver.class);
      job.setMapperClass(MyDriverMapper.class);
      **job.setCombinerClass(MyDriverReducer.class);**
      job.setReducerClass(MyDriverReducer.class);
      job.setMapOutputKeyClass(Text.class);
      job.setMapOutputValueClass(LongWritable.class);
      job.setOutputKeyClass(Text.class);
      job.setOutputValueClass(Text.class);
      job.setInputFormatClass(TextInputFormat.class);
      FileInputFormat.addInputPath(job, new Path(args[1]));
      FileOutputFormat.setOutputPath(job, new Path(args[2]));
      return (job.waitForCompletion(true) ? 0 : 1);
     }
    }

Outcomes