View Javadoc

1   /**
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one or more
5    * contributor license agreements. See the NOTICE file distributed with this
6    * work for additional information regarding copyright ownership. The ASF
7    * licenses this file to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance with the License.
9    * You may obtain a copy of the License at
10   *
11   * http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
15   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
16   * License for the specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertTrue;
24  import static org.junit.Assert.fail;
25  
26  import java.io.IOException;
27  import java.util.ArrayList;
28  import java.util.List;
29  
30  import org.apache.commons.lang.math.RandomUtils;
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.hbase.HBaseTestingUtility;
36  import org.apache.hadoop.hbase.HConstants;
37  import org.apache.hadoop.hbase.HRegionInfo;
38  import org.apache.hadoop.hbase.HTableDescriptor;
39  import org.apache.hadoop.hbase.testclassification.LargeTests;
40  import org.apache.hadoop.hbase.MiniHBaseCluster;
41  import org.apache.hadoop.hbase.ServerName;
42  import org.apache.hadoop.hbase.TableName;
43  import org.apache.hadoop.hbase.UnknownRegionException;
44  import org.apache.hadoop.hbase.MetaTableAccessor;
45  import org.apache.hadoop.hbase.client.Admin;
46  import org.apache.hadoop.hbase.client.Put;
47  import org.apache.hadoop.hbase.client.RegionReplicaUtil;
48  import org.apache.hadoop.hbase.client.Result;
49  import org.apache.hadoop.hbase.client.ResultScanner;
50  import org.apache.hadoop.hbase.client.Scan;
51  import org.apache.hadoop.hbase.client.Table;
52  import org.apache.hadoop.hbase.exceptions.MergeRegionException;
53  import org.apache.hadoop.hbase.master.AssignmentManager;
54  import org.apache.hadoop.hbase.master.HMaster;
55  import org.apache.hadoop.hbase.master.RegionState.State;
56  import org.apache.hadoop.hbase.master.RegionStates;
57  import org.apache.hadoop.hbase.util.Bytes;
58  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
59  import org.apache.hadoop.hbase.util.FSUtils;
60  import org.apache.hadoop.hbase.util.Pair;
61  import org.apache.hadoop.hbase.util.PairOfSameType;
62  import org.apache.hadoop.util.StringUtils;
63  import org.junit.AfterClass;
64  import org.junit.BeforeClass;
65  import org.junit.Test;
66  import org.junit.experimental.categories.Category;
67  
68  import com.google.common.base.Joiner;
69  
70  /**
71   * Like {@link TestRegionMergeTransaction} in that we're testing
72   * {@link RegionMergeTransactionImpl} only the below tests are against a running
73   * cluster where {@link TestRegionMergeTransaction} is tests against bare
74   * {@link HRegion}.
75   */
76  @Category(LargeTests.class)
77  public class TestRegionMergeTransactionOnCluster {
78    private static final Log LOG = LogFactory
79        .getLog(TestRegionMergeTransactionOnCluster.class);
80    private static final int NB_SERVERS = 3;
81  
82    private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
83    private static final byte[] QUALIFIER = Bytes.toBytes("q");
84  
85    private static byte[] ROW = Bytes.toBytes("testRow");
86    private static final int INITIAL_REGION_NUM = 10;
87    private static final int ROWSIZE = 200;
88    private static byte[][] ROWS = makeN(ROW, ROWSIZE);
89  
90    private static int waitTime = 60 * 1000;
91  
92    static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
93  
94    private static HMaster master;
95    private static Admin admin;
96  
97    static void setupOnce() throws Exception {
98      // Start a cluster
99      TEST_UTIL.startMiniCluster(NB_SERVERS);
100     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
101     master = cluster.getMaster();
102     master.balanceSwitch(false);
103     admin = TEST_UTIL.getHBaseAdmin();
104   }
105 
106   @BeforeClass
107   public static void beforeAllTests() throws Exception {
108     // Use ZK for region assignment
109     TEST_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
110     setupOnce();
111   }
112 
113   @AfterClass
114   public static void afterAllTests() throws Exception {
115     TEST_UTIL.shutdownMiniCluster();
116   }
117 
118   @Test
119   public void testWholesomeMerge() throws Exception {
120     LOG.info("Starting testWholesomeMerge");
121     final TableName tableName =
122         TableName.valueOf("testWholesomeMerge");
123 
124     // Create table and load data.
125     Table table = createTableAndLoadData(master, tableName);
126     // Merge 1st and 2nd region
127     mergeRegionsAndVerifyRegionNum(master, tableName, 0, 1,
128         INITIAL_REGION_NUM - 1);
129 
130     // Merge 2nd and 3th region
131     PairOfSameType<HRegionInfo> mergedRegions =
132       mergeRegionsAndVerifyRegionNum(master, tableName, 1, 2,
133         INITIAL_REGION_NUM - 2);
134 
135     verifyRowCount(table, ROWSIZE);
136 
137     // Randomly choose one of the two merged regions
138     HRegionInfo hri = RandomUtils.nextBoolean() ?
139       mergedRegions.getFirst() : mergedRegions.getSecond();
140     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
141     AssignmentManager am = cluster.getMaster().getAssignmentManager();
142     RegionStates regionStates = am.getRegionStates();
143     long start = EnvironmentEdgeManager.currentTime();
144     while (!regionStates.isRegionInState(hri, State.MERGED)) {
145       assertFalse("Timed out in waiting one merged region to be in state MERGED",
146         EnvironmentEdgeManager.currentTime() - start > 60000);
147       Thread.sleep(500);
148     }
149 
150     // We should not be able to assign it again
151     am.assign(hri, true, true);
152     assertFalse("Merged region can't be assigned",
153       regionStates.isRegionInTransition(hri));
154     assertTrue(regionStates.isRegionInState(hri, State.MERGED));
155 
156     // We should not be able to unassign it either
157     am.unassign(hri, true, null);
158     assertFalse("Merged region can't be unassigned",
159       regionStates.isRegionInTransition(hri));
160     assertTrue(regionStates.isRegionInState(hri, State.MERGED));
161 
162     table.close();
163   }
164 
165   @Test
166   public void testCleanMergeReference() throws Exception {
167     LOG.info("Starting testCleanMergeReference");
168     admin.enableCatalogJanitor(false);
169     try {
170       final TableName tableName =
171           TableName.valueOf("testCleanMergeReference");
172       // Create table and load data.
173       Table table = createTableAndLoadData(master, tableName);
174       // Merge 1st and 2nd region
175       mergeRegionsAndVerifyRegionNum(master, tableName, 0, 1,
176           INITIAL_REGION_NUM - 1);
177       verifyRowCount(table, ROWSIZE);
178       table.close();
179 
180       List<Pair<HRegionInfo, ServerName>> tableRegions = MetaTableAccessor
181           .getTableRegionsAndLocations(master.getZooKeeper(), master.getConnection(), tableName);
182       HRegionInfo mergedRegionInfo = tableRegions.get(0).getFirst();
183       HTableDescriptor tableDescritor = master.getTableDescriptors().get(
184           tableName);
185       Result mergedRegionResult = MetaTableAccessor.getRegionResult(
186         master.getConnection(), mergedRegionInfo.getRegionName());
187 
188       // contains merge reference in META
189       assertTrue(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY,
190           HConstants.MERGEA_QUALIFIER) != null);
191       assertTrue(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY,
192           HConstants.MERGEB_QUALIFIER) != null);
193 
194       // merging regions' directory are in the file system all the same
195       HRegionInfo regionA = HRegionInfo.getHRegionInfo(mergedRegionResult,
196           HConstants.MERGEA_QUALIFIER);
197       HRegionInfo regionB = HRegionInfo.getHRegionInfo(mergedRegionResult,
198           HConstants.MERGEB_QUALIFIER);
199       FileSystem fs = master.getMasterFileSystem().getFileSystem();
200       Path rootDir = master.getMasterFileSystem().getRootDir();
201 
202       Path tabledir = FSUtils.getTableDir(rootDir, mergedRegionInfo.getTable());
203       Path regionAdir = new Path(tabledir, regionA.getEncodedName());
204       Path regionBdir = new Path(tabledir, regionB.getEncodedName());
205       assertTrue(fs.exists(regionAdir));
206       assertTrue(fs.exists(regionBdir));
207 
208       admin.compactRegion(mergedRegionInfo.getRegionName());
209       // wait until merged region doesn't have reference file
210       long timeout = System.currentTimeMillis() + waitTime;
211       HRegionFileSystem hrfs = new HRegionFileSystem(
212           TEST_UTIL.getConfiguration(), fs, tabledir, mergedRegionInfo);
213       while (System.currentTimeMillis() < timeout) {
214         if (!hrfs.hasReferences(tableDescritor)) {
215           break;
216         }
217         Thread.sleep(50);
218       }
219       assertFalse(hrfs.hasReferences(tableDescritor));
220 
221       // run CatalogJanitor to clean merge references in hbase:meta and archive the
222       // files of merging regions
223       int cleaned = admin.runCatalogScan();
224       assertTrue(cleaned > 0);
225       assertFalse(fs.exists(regionAdir));
226       assertFalse(fs.exists(regionBdir));
227 
228       mergedRegionResult = MetaTableAccessor.getRegionResult(
229         master.getConnection(), mergedRegionInfo.getRegionName());
230       assertFalse(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY,
231           HConstants.MERGEA_QUALIFIER) != null);
232       assertFalse(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY,
233           HConstants.MERGEB_QUALIFIER) != null);
234 
235     } finally {
236       admin.enableCatalogJanitor(true);
237     }
238   }
239 
240   /**
241    * This test tests 1, merging region not online;
242    * 2, merging same two regions; 3, merging unknown regions.
243    * They are in one test case so that we don't have to create
244    * many tables, and these tests are simple.
245    */
246   @Test
247   public void testMerge() throws Exception {
248     LOG.info("Starting testMerge");
249     final TableName tableName = TableName.valueOf("testMerge");
250 
251     try {
252       // Create table and load data.
253       Table table = createTableAndLoadData(master, tableName);
254       RegionStates regionStates = master.getAssignmentManager().getRegionStates();
255       List<HRegionInfo> regions = regionStates.getRegionsOfTable(tableName);
256       // Fake offline one region
257       HRegionInfo a = regions.get(0);
258       HRegionInfo b = regions.get(1);
259       regionStates.regionOffline(a);
260       try {
261         // Merge offline region. Region a is offline here
262         admin.mergeRegions(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false);
263         fail("Offline regions should not be able to merge");
264       } catch (IOException ie) {
265         System.out.println(ie);
266         assertTrue("Exception should mention regions not online",
267           StringUtils.stringifyException(ie).contains("regions not online")
268             && ie instanceof MergeRegionException);
269       }
270       try {
271         // Merge the same region: b and b.
272         admin.mergeRegions(b.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), true);
273         fail("A region should not be able to merge with itself, even forcifully");
274       } catch (IOException ie) {
275         assertTrue("Exception should mention regions not online",
276           StringUtils.stringifyException(ie).contains("region to itself")
277             && ie instanceof MergeRegionException);
278       }
279       try {
280         // Merge unknown regions
281         admin.mergeRegions(Bytes.toBytes("-f1"), Bytes.toBytes("-f2"), true);
282         fail("Unknown region could not be merged");
283       } catch (IOException ie) {
284         assertTrue("UnknownRegionException should be thrown",
285           ie instanceof UnknownRegionException);
286       }
287       table.close();
288     } finally {
289       TEST_UTIL.deleteTable(tableName);
290     }
291   }
292 
293   @Test
294   public void testMergeWithReplicas() throws Exception {
295     final TableName tableName = TableName.valueOf("testMergeWithReplicas");
296     // Create table and load data.
297     createTableAndLoadData(master, tableName, 5, 2);
298     List<Pair<HRegionInfo, ServerName>> initialRegionToServers =
299         MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(), master.getConnection(),
300            tableName);
301     // Merge 1st and 2nd region
302     PairOfSameType<HRegionInfo> mergedRegions = mergeRegionsAndVerifyRegionNum(master, tableName,
303         0, 2, 5 * 2 - 2);
304     List<Pair<HRegionInfo, ServerName>> currentRegionToServers =
305         MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(), master.getConnection(),
306            tableName);
307     List<HRegionInfo> initialRegions = new ArrayList<HRegionInfo>();
308     for (Pair<HRegionInfo, ServerName> p : initialRegionToServers) {
309       initialRegions.add(p.getFirst());
310     }
311     List<HRegionInfo> currentRegions = new ArrayList<HRegionInfo>();
312     for (Pair<HRegionInfo, ServerName> p : currentRegionToServers) {
313       currentRegions.add(p.getFirst());
314     }
315     assertTrue(initialRegions.contains(mergedRegions.getFirst())); //this is the first region
316     assertTrue(initialRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
317         mergedRegions.getFirst(), 1))); //this is the replica of the first region
318     assertTrue(initialRegions.contains(mergedRegions.getSecond())); //this is the second region
319     assertTrue(initialRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
320         mergedRegions.getSecond(), 1))); //this is the replica of the second region
321     assertTrue(!initialRegions.contains(currentRegions.get(0))); //this is the new region
322     assertTrue(!initialRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
323         currentRegions.get(0), 1))); //replica of the new region
324     assertTrue(currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
325         currentRegions.get(0), 1))); //replica of the new region
326     assertTrue(!currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
327         mergedRegions.getFirst(), 1))); //replica of the merged region
328     assertTrue(!currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
329         mergedRegions.getSecond(), 1))); //replica of the merged region
330   }
331 
332   private PairOfSameType<HRegionInfo> mergeRegionsAndVerifyRegionNum(
333       HMaster master, TableName tablename,
334       int regionAnum, int regionBnum, int expectedRegionNum) throws Exception {
335     PairOfSameType<HRegionInfo> mergedRegions =
336       requestMergeRegion(master, tablename, regionAnum, regionBnum);
337     waitAndVerifyRegionNum(master, tablename, expectedRegionNum);
338     return mergedRegions;
339   }
340 
341   private PairOfSameType<HRegionInfo> requestMergeRegion(
342       HMaster master, TableName tablename,
343       int regionAnum, int regionBnum) throws Exception {
344     List<Pair<HRegionInfo, ServerName>> tableRegions = MetaTableAccessor
345         .getTableRegionsAndLocations(master.getZooKeeper(),
346           master.getConnection(), tablename);
347     HRegionInfo regionA = tableRegions.get(regionAnum).getFirst();
348     HRegionInfo regionB = tableRegions.get(regionBnum).getFirst();
349     TEST_UTIL.getHBaseAdmin().mergeRegions(
350       regionA.getEncodedNameAsBytes(),
351       regionB.getEncodedNameAsBytes(), false);
352     return new PairOfSameType<HRegionInfo>(regionA, regionB);
353   }
354 
355   private void waitAndVerifyRegionNum(HMaster master, TableName tablename,
356       int expectedRegionNum) throws Exception {
357     List<Pair<HRegionInfo, ServerName>> tableRegionsInMeta;
358     List<HRegionInfo> tableRegionsInMaster;
359     long timeout = System.currentTimeMillis() + waitTime;
360     while (System.currentTimeMillis() < timeout) {
361       tableRegionsInMeta = MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(),
362         master.getConnection(), tablename);
363       tableRegionsInMaster = master.getAssignmentManager().getRegionStates()
364           .getRegionsOfTable(tablename);
365       if (tableRegionsInMeta.size() == expectedRegionNum
366           && tableRegionsInMaster.size() == expectedRegionNum) {
367         break;
368       }
369       Thread.sleep(250);
370     }
371 
372     tableRegionsInMeta = MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(),
373       master.getConnection(), tablename);
374     LOG.info("Regions after merge:" + Joiner.on(',').join(tableRegionsInMeta));
375     assertEquals(expectedRegionNum, tableRegionsInMeta.size());
376   }
377 
378   private Table createTableAndLoadData(HMaster master, TableName tablename)
379       throws Exception {
380     return createTableAndLoadData(master, tablename, INITIAL_REGION_NUM, 1);
381   }
382 
383   private Table createTableAndLoadData(HMaster master, TableName tablename,
384       int numRegions, int replication) throws Exception {
385     assertTrue("ROWSIZE must > numregions:" + numRegions, ROWSIZE > numRegions);
386     byte[][] splitRows = new byte[numRegions - 1][];
387     for (int i = 0; i < splitRows.length; i++) {
388       splitRows[i] = ROWS[(i + 1) * ROWSIZE / numRegions];
389     }
390 
391     Table table = TEST_UTIL.createTable(tablename, FAMILYNAME, splitRows);
392     if (replication > 1) {
393       HBaseTestingUtility.setReplicas(admin, tablename, replication);
394     }
395     loadData(table);
396     verifyRowCount(table, ROWSIZE);
397 
398     // sleep here is an ugly hack to allow region transitions to finish
399     long timeout = System.currentTimeMillis() + waitTime;
400     List<Pair<HRegionInfo, ServerName>> tableRegions;
401     while (System.currentTimeMillis() < timeout) {
402       tableRegions = MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(),
403         master.getConnection(), tablename);
404       if (tableRegions.size() == numRegions * replication)
405         break;
406       Thread.sleep(250);
407     }
408 
409     tableRegions = MetaTableAccessor.getTableRegionsAndLocations(
410       master.getZooKeeper(),
411       master.getConnection(), tablename);
412     LOG.info("Regions after load: " + Joiner.on(',').join(tableRegions));
413     assertEquals(numRegions * replication, tableRegions.size());
414     return table;
415   }
416 
417   private static byte[][] makeN(byte[] base, int n) {
418     byte[][] ret = new byte[n][];
419     for (int i = 0; i < n; i++) {
420       ret[i] = Bytes.add(base, Bytes.toBytes(String.format("%04d", i)));
421     }
422     return ret;
423   }
424 
425   private void loadData(Table table) throws IOException {
426     for (int i = 0; i < ROWSIZE; i++) {
427       Put put = new Put(ROWS[i]);
428       put.add(FAMILYNAME, QUALIFIER, Bytes.toBytes(i));
429       table.put(put);
430     }
431   }
432 
433   private void verifyRowCount(Table table, int expectedRegionNum)
434       throws IOException {
435     ResultScanner scanner = table.getScanner(new Scan());
436     int rowCount = 0;
437     while (scanner.next() != null) {
438       rowCount++;
439     }
440     assertEquals(expectedRegionNum, rowCount);
441     scanner.close();
442   }
443 }