Skip to content

Commit

Permalink
Account for different block size variable
Browse files Browse the repository at this point in the history
  • Loading branch information
edwardcapriolo committed Jul 25, 2014
1 parent 591f1b8 commit f11228f
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 3 deletions.
19 changes: 18 additions & 1 deletion src/main/java/com/m6d/filecrush/crush/Crush.java
Original file line number Diff line number Diff line change
Expand Up @@ -488,7 +488,7 @@ boolean createJobConfAndParseArgs(String... args) throws ParseException, IOExcep
}
}

dfsBlockSize = Long.parseLong(job.get("dfs.block.size"));
dfsBlockSize = parseDfsBlockSize(job);
maxEligibleSize = (long) (dfsBlockSize * threshold);
}

Expand Down Expand Up @@ -589,6 +589,23 @@ boolean createJobConfAndParseArgs(String... args) throws ParseException, IOExcep
return true;
}

/**
* The block size has changed over the years... Get with the times.
* @param job a conf to check for data
* @return a long representing block size
* @throws RuntimeException if can not determine block size
*/
private long parseDfsBlockSize(JobConf job){
long old = job.getLong("dfs.block.size", -1);
if (old == -1){
old = job.getLong("dfs.blocksize", -1);
}
if (old == -1){
throw new RuntimeException ( "Could not determine how to set block size. Abandon ship!");
}
return old;
}

@Override
public int run(String[] args) throws Exception {

Expand Down
5 changes: 3 additions & 2 deletions src/test/java/com/m6d/filecrush/crush/CrushTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -713,12 +713,13 @@ public void bucketing() throws Exception {
*/
Map<String, Integer> expectedPartitions = new HashMap<String, Integer>();

//TODO: this may not be deterministic due to jvm/hashmap/filesystem
expectedPartitions.put(dir2.getAbsolutePath() + "-1", 0);
expectedPartitions.put(dir2_4_2.getAbsolutePath() + "-0", 1);
expectedPartitions.put(dir1_1.getAbsolutePath() + "-0", 2);
expectedPartitions.put(dir1_1.getAbsolutePath() + "-2", 3);
expectedPartitions.put(dir1_1.getAbsolutePath() + "-2", 4);
expectedPartitions.put(dir2_2.getAbsolutePath() + "-1", 3);
expectedPartitions.put(dir1_1.getAbsolutePath() + "-1", 4);
expectedPartitions.put(dir1_1.getAbsolutePath() + "-1", 3);
expectedPartitions.put(dir1_2.getAbsolutePath() + "-0", 4);

assertThat(actualPartitions, equalTo(expectedPartitions));
Expand Down

0 comments on commit f11228f

Please sign in to comment.