View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.util;
20  
21  import static org.junit.Assert.assertArrayEquals;
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertFalse;
24  import static org.junit.Assert.assertNotSame;
25  import static org.junit.Assert.assertTrue;
26  
27  import java.util.ArrayList;
28  import java.util.List;
29  import java.util.Map;
30  
31  import org.apache.commons.lang.ArrayUtils;
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.hbase.HBaseTestingUtility;
36  import org.apache.hadoop.hbase.HRegionInfo;
37  import org.apache.hadoop.hbase.testclassification.MediumTests;
38  import org.apache.hadoop.hbase.ServerName;
39  import org.apache.hadoop.hbase.TableName;
40  import org.apache.hadoop.hbase.client.HTable;
41  import org.apache.hadoop.hbase.util.RegionSplitter.HexStringSplit;
42  import org.apache.hadoop.hbase.util.RegionSplitter.SplitAlgorithm;
43  import org.apache.hadoop.hbase.util.RegionSplitter.UniformSplit;
44  import org.junit.AfterClass;
45  import org.junit.BeforeClass;
46  import org.junit.Test;
47  import org.junit.experimental.categories.Category;
48  
49  /**
50   * Tests for {@link RegionSplitter}, which can create a pre-split table or do a
51   * rolling split of an existing table.
52   */
53  @Category(MediumTests.class)
54  public class TestRegionSplitter {
55      private final static Log LOG = LogFactory.getLog(TestRegionSplitter.class);
56      private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
57      private final static String CF_NAME = "SPLIT_TEST_CF";
58      private final static byte xFF = (byte) 0xff;
59  
60      @BeforeClass
61      public static void setup() throws Exception {
62          UTIL.startMiniCluster();
63      }
64  
65      @AfterClass
66      public static void teardown() throws Exception {
67          UTIL.shutdownMiniCluster();
68      }
69  
70      /**
71       * Test creating a pre-split table using the HexStringSplit algorithm.
72       */
73      @Test
74      public void testCreatePresplitTableHex() throws Exception {
75        final List<byte[]> expectedBounds = new ArrayList<byte[]>();
76        expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY);
77        expectedBounds.add("10000000".getBytes());
78        expectedBounds.add("20000000".getBytes());
79        expectedBounds.add("30000000".getBytes());
80        expectedBounds.add("40000000".getBytes());
81        expectedBounds.add("50000000".getBytes());
82        expectedBounds.add("60000000".getBytes());
83        expectedBounds.add("70000000".getBytes());
84        expectedBounds.add("80000000".getBytes());
85        expectedBounds.add("90000000".getBytes());
86        expectedBounds.add("a0000000".getBytes());
87        expectedBounds.add("b0000000".getBytes());
88        expectedBounds.add("c0000000".getBytes());
89        expectedBounds.add("d0000000".getBytes());
90        expectedBounds.add("e0000000".getBytes());
91        expectedBounds.add("f0000000".getBytes());
92            expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY);
93  
94            // Do table creation/pre-splitting and verification of region boundaries
95      preSplitTableAndVerify(expectedBounds,
96          HexStringSplit.class.getSimpleName(),
97          TableName.valueOf("NewHexPresplitTable"));
98      }
99  
100     /**
101      * Test creating a pre-split table using the UniformSplit algorithm.
102      */
103     @Test
104     public void testCreatePresplitTableUniform() throws Exception {
105       List<byte[]> expectedBounds = new ArrayList<byte[]>();
106       expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY);
107       expectedBounds.add(new byte[] {      0x10, 0, 0, 0, 0, 0, 0, 0});
108       expectedBounds.add(new byte[] {      0x20, 0, 0, 0, 0, 0, 0, 0});
109       expectedBounds.add(new byte[] {      0x30, 0, 0, 0, 0, 0, 0, 0});
110       expectedBounds.add(new byte[] {      0x40, 0, 0, 0, 0, 0, 0, 0});
111       expectedBounds.add(new byte[] {      0x50, 0, 0, 0, 0, 0, 0, 0});
112       expectedBounds.add(new byte[] {      0x60, 0, 0, 0, 0, 0, 0, 0});
113       expectedBounds.add(new byte[] {      0x70, 0, 0, 0, 0, 0, 0, 0});
114       expectedBounds.add(new byte[] {(byte)0x80, 0, 0, 0, 0, 0, 0, 0});
115       expectedBounds.add(new byte[] {(byte)0x90, 0, 0, 0, 0, 0, 0, 0});
116       expectedBounds.add(new byte[] {(byte)0xa0, 0, 0, 0, 0, 0, 0, 0});
117       expectedBounds.add(new byte[] {(byte)0xb0, 0, 0, 0, 0, 0, 0, 0});
118       expectedBounds.add(new byte[] {(byte)0xc0, 0, 0, 0, 0, 0, 0, 0});
119       expectedBounds.add(new byte[] {(byte)0xd0, 0, 0, 0, 0, 0, 0, 0});
120       expectedBounds.add(new byte[] {(byte)0xe0, 0, 0, 0, 0, 0, 0, 0});
121       expectedBounds.add(new byte[] {(byte)0xf0, 0, 0, 0, 0, 0, 0, 0});
122       expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY);
123 
124       // Do table creation/pre-splitting and verification of region boundaries
125       preSplitTableAndVerify(expectedBounds, UniformSplit.class.getSimpleName(),
126         TableName.valueOf("NewUniformPresplitTable"));
127     }
128 
129     /**
130      * Unit tests for the HexStringSplit algorithm. Makes sure it divides up the
131      * space of keys in the way that we expect.
132      */
133     @Test
134     public void unitTestHexStringSplit() {
135         HexStringSplit splitter = new HexStringSplit();
136         // Check splitting while starting from scratch
137 
138         byte[][] twoRegionsSplits = splitter.split(2);
139         assertEquals(1, twoRegionsSplits.length);
140     assertArrayEquals(twoRegionsSplits[0], "80000000".getBytes());
141 
142         byte[][] threeRegionsSplits = splitter.split(3);
143         assertEquals(2, threeRegionsSplits.length);
144         byte[] expectedSplit0 = "55555555".getBytes();
145         assertArrayEquals(expectedSplit0, threeRegionsSplits[0]);
146         byte[] expectedSplit1 = "aaaaaaaa".getBytes();
147         assertArrayEquals(expectedSplit1, threeRegionsSplits[1]);
148 
149         // Check splitting existing regions that have start and end points
150         byte[] splitPoint = splitter.split("10000000".getBytes(), "30000000".getBytes());
151         assertArrayEquals("20000000".getBytes(), splitPoint);
152 
153         byte[] lastRow = "ffffffff".getBytes();
154         assertArrayEquals(lastRow, splitter.lastRow());
155         byte[] firstRow = "00000000".getBytes();
156         assertArrayEquals(firstRow, splitter.firstRow());
157 
158         // Halfway between 00... and 20... should be 10...
159         splitPoint = splitter.split(firstRow, "20000000".getBytes());
160         assertArrayEquals(splitPoint, "10000000".getBytes());
161 
162         // Halfway between df... and ff... should be ef....
163         splitPoint = splitter.split("dfffffff".getBytes(), lastRow);
164         assertArrayEquals(splitPoint,"efffffff".getBytes());
165     }
166 
167     /**
168      * Unit tests for the UniformSplit algorithm. Makes sure it divides up the space of
169      * keys in the way that we expect.
170      */
171     @Test
172     public void unitTestUniformSplit() {
173         UniformSplit splitter = new UniformSplit();
174 
175         // Check splitting while starting from scratch
176         try {
177             splitter.split(1);
178             throw new AssertionError("Splitting into <2 regions should have thrown exception");
179         } catch (IllegalArgumentException e) { }
180 
181         byte[][] twoRegionsSplits = splitter.split(2);
182         assertEquals(1, twoRegionsSplits.length);
183         assertArrayEquals(twoRegionsSplits[0],
184             new byte[] { (byte) 0x80, 0, 0, 0, 0, 0, 0, 0 });
185 
186         byte[][] threeRegionsSplits = splitter.split(3);
187         assertEquals(2, threeRegionsSplits.length);
188         byte[] expectedSplit0 = new byte[] {0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55};
189         assertArrayEquals(expectedSplit0, threeRegionsSplits[0]);
190         byte[] expectedSplit1 = new byte[] {(byte)0xAA, (byte)0xAA, (byte)0xAA, (byte)0xAA,
191                 (byte)0xAA, (byte)0xAA, (byte)0xAA, (byte)0xAA};
192         assertArrayEquals(expectedSplit1, threeRegionsSplits[1]);
193 
194         // Check splitting existing regions that have start and end points
195         byte[] splitPoint = splitter.split(new byte[] {0x10}, new byte[] {0x30});
196         assertArrayEquals(new byte[] {0x20}, splitPoint);
197 
198         byte[] lastRow = new byte[] {xFF, xFF, xFF, xFF, xFF, xFF, xFF, xFF};
199         assertArrayEquals(lastRow, splitter.lastRow());
200         byte[] firstRow = ArrayUtils.EMPTY_BYTE_ARRAY;
201         assertArrayEquals(firstRow, splitter.firstRow());
202 
203         splitPoint = splitter.split(firstRow, new byte[] {0x20});
204         assertArrayEquals(splitPoint, new byte[] {0x10});
205 
206         splitPoint = splitter.split(new byte[] {(byte)0xdf, xFF, xFF, xFF, xFF,
207                 xFF, xFF, xFF}, lastRow);
208         assertArrayEquals(splitPoint,
209                 new byte[] {(byte)0xef, xFF, xFF, xFF, xFF, xFF, xFF, xFF});
210 
211         splitPoint = splitter.split(new byte[] {'a', 'a', 'a'}, new byte[] {'a', 'a', 'b'});
212         assertArrayEquals(splitPoint, new byte[] {'a', 'a', 'a', (byte)0x80 });
213     }
214 
215   @Test
216   public void testUserInput() {
217     SplitAlgorithm algo = new HexStringSplit();
218     assertFalse(splitFailsPrecondition(algo)); // default settings are fine
219     assertFalse(splitFailsPrecondition(algo, "00", "AA")); // custom is fine
220     assertTrue(splitFailsPrecondition(algo, "AA", "00")); // range error
221     assertTrue(splitFailsPrecondition(algo, "AA", "AA")); // range error
222     assertFalse(splitFailsPrecondition(algo, "0", "2", 3)); // should be fine
223     assertFalse(splitFailsPrecondition(algo, "0", "A", 11)); // should be fine
224     assertTrue(splitFailsPrecondition(algo, "0", "A", 12)); // too granular
225 
226     algo = new UniformSplit();
227     assertFalse(splitFailsPrecondition(algo)); // default settings are fine
228     assertFalse(splitFailsPrecondition(algo, "\\x00", "\\xAA")); // custom is fine
229     assertTrue(splitFailsPrecondition(algo, "\\xAA", "\\x00")); // range error
230     assertTrue(splitFailsPrecondition(algo, "\\xAA", "\\xAA")); // range error
231     assertFalse(splitFailsPrecondition(algo, "\\x00", "\\x02", 3)); // should be fine
232     assertFalse(splitFailsPrecondition(algo, "\\x00", "\\x0A", 11)); // should be fine
233     assertFalse(splitFailsPrecondition(algo, "\\x00", "\\x0A", 12)); // should be fine
234   }
235 
236   private boolean splitFailsPrecondition(SplitAlgorithm algo) {
237     return splitFailsPrecondition(algo, 100);
238   }
239 
240   private boolean splitFailsPrecondition(SplitAlgorithm algo, String firstRow,
241       String lastRow) {
242     return splitFailsPrecondition(algo, firstRow, lastRow, 100);
243   }
244 
245   private boolean splitFailsPrecondition(SplitAlgorithm algo, String firstRow,
246       String lastRow, int numRegions) {
247     algo.setFirstRow(firstRow);
248     algo.setLastRow(lastRow);
249     return splitFailsPrecondition(algo, numRegions);
250   }
251 
252   private boolean splitFailsPrecondition(SplitAlgorithm algo, int numRegions) {
253     try {
254       byte[][] s = algo.split(numRegions);
255       LOG.debug("split algo = " + algo);
256       if (s != null) {
257         StringBuilder sb = new StringBuilder();
258         for (byte[] b : s) {
259           sb.append(Bytes.toStringBinary(b) + "  ");
260         }
261         LOG.debug(sb.toString());
262       }
263       return false;
264     } catch (IllegalArgumentException e) {
265       return true;
266     } catch (IllegalStateException e) {
267       return true;
268     } catch (IndexOutOfBoundsException e) {
269       return true;
270     }
271   }
272 
273     /**
274      * Creates a pre-split table with expectedBounds.size()+1 regions, then
275      * verifies that the region boundaries are the same as the expected
276      * region boundaries in expectedBounds.
277      * @throws Various junit assertions
278      */
279     private void preSplitTableAndVerify(List<byte[]> expectedBounds,
280             String splitClass, TableName tableName) throws Exception {
281         final int numRegions = expectedBounds.size()-1;
282         final Configuration conf = UTIL.getConfiguration();
283         conf.setInt("split.count", numRegions);
284         SplitAlgorithm splitAlgo = RegionSplitter.newSplitAlgoInstance(conf, splitClass);
285         RegionSplitter.createPresplitTable(tableName, splitAlgo, new String[] {CF_NAME}, conf);
286         verifyBounds(expectedBounds, tableName);
287     }
288 
289   @Test
290   public void noopRollingSplit() throws Exception {
291     final List<byte[]> expectedBounds = new ArrayList<byte[]>();
292     expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY);
293     rollingSplitAndVerify(
294         TableName.valueOf(TestRegionSplitter.class.getSimpleName()),
295         "UniformSplit", expectedBounds);
296   }
297 
298     private void rollingSplitAndVerify(TableName tableName, String splitClass,
299             List<byte[]> expectedBounds)  throws Exception {
300         final Configuration conf = UTIL.getConfiguration();
301 
302         // Set this larger than the number of splits so RegionSplitter won't block
303         conf.setInt("split.outstanding", 5);
304         SplitAlgorithm splitAlgo = RegionSplitter.newSplitAlgoInstance(conf, splitClass);
305         RegionSplitter.rollingSplit(tableName, splitAlgo, conf);
306         verifyBounds(expectedBounds, tableName);
307     }
308 
309     private void verifyBounds(List<byte[]> expectedBounds, TableName tableName)
310             throws Exception {
311         // Get region boundaries from the cluster and verify their endpoints
312         final Configuration conf = UTIL.getConfiguration();
313         final int numRegions = expectedBounds.size()-1;
314         final HTable hTable = new HTable(conf, tableName);
315         final Map<HRegionInfo, ServerName> regionInfoMap = hTable.getRegionLocations();
316         assertEquals(numRegions, regionInfoMap.size());
317         for (Map.Entry<HRegionInfo, ServerName> entry: regionInfoMap.entrySet()) {
318             final HRegionInfo regionInfo = entry.getKey();
319             byte[] regionStart = regionInfo.getStartKey();
320             byte[] regionEnd = regionInfo.getEndKey();
321 
322             // This region's start key should be one of the region boundaries
323             int startBoundaryIndex = indexOfBytes(expectedBounds, regionStart);
324             assertNotSame(-1, startBoundaryIndex);
325 
326             // This region's end key should be the region boundary that comes
327             // after the starting boundary.
328             byte[] expectedRegionEnd = expectedBounds.get(
329                     startBoundaryIndex+1);
330             assertEquals(0, Bytes.compareTo(regionEnd, expectedRegionEnd));
331         }
332         hTable.close();
333     }
334 
335     /**
336      * List.indexOf() doesn't really work for a List<byte[]>, because byte[]
337      * doesn't override equals(). This method checks whether a list contains
338      * a given element by checking each element using the byte array
339      * comparator.
340      * @return the index of the first element that equals compareTo, or -1
341      * if no elements are equal.
342      */
343     static private int indexOfBytes(List<byte[]> list,  byte[] compareTo) {
344         int listIndex = 0;
345         for(byte[] elem: list) {
346             if(Bytes.BYTES_COMPARATOR.compare(elem, compareTo) == 0) {
347                 return listIndex;
348             }
349             listIndex++;
350         }
351         return -1;
352     }
353 
354 }
355