1
2
3
4
5
6
7
8
9
10
11 package org.apache.hadoop.hbase.namespace;
12
13 import static org.junit.Assert.assertEquals;
14 import static org.junit.Assert.assertFalse;
15 import static org.junit.Assert.assertNotNull;
16 import static org.junit.Assert.assertNull;
17 import static org.junit.Assert.assertTrue;
18 import static org.junit.Assert.fail;
19
20 import java.io.IOException;
21 import java.util.Collections;
22 import java.util.List;
23 import java.util.Set;
24 import java.util.concurrent.CountDownLatch;
25
26 import org.apache.commons.lang.StringUtils;
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.FileSystem;
31 import org.apache.hadoop.fs.Path;
32 import org.apache.hadoop.hbase.CategoryBasedTimeout;
33 import org.apache.hadoop.hbase.Coprocessor;
34 import org.apache.hadoop.hbase.CoprocessorEnvironment;
35 import org.apache.hadoop.hbase.DoNotRetryIOException;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HColumnDescriptor;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.HRegionInfo;
40 import org.apache.hadoop.hbase.HTableDescriptor;
41 import org.apache.hadoop.hbase.MiniHBaseCluster;
42 import org.apache.hadoop.hbase.NamespaceDescriptor;
43 import org.apache.hadoop.hbase.TableName;
44 import org.apache.hadoop.hbase.Waiter;
45 import org.apache.hadoop.hbase.client.Connection;
46 import org.apache.hadoop.hbase.client.ConnectionFactory;
47 import org.apache.hadoop.hbase.client.HBaseAdmin;
48 import org.apache.hadoop.hbase.client.HTable;
49 import org.apache.hadoop.hbase.client.Mutation;
50 import org.apache.hadoop.hbase.client.RegionLocator;
51 import org.apache.hadoop.hbase.client.Table;
52 import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
53 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
54 import org.apache.hadoop.hbase.coprocessor.BaseRegionServerObserver;
55 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
56 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
57 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
58 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
59 import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
60 import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
61 import org.apache.hadoop.hbase.mapreduce.TableInputFormatBase;
62 import org.apache.hadoop.hbase.master.HMaster;
63 import org.apache.hadoop.hbase.master.RegionState;
64 import org.apache.hadoop.hbase.master.RegionStates;
65 import org.apache.hadoop.hbase.master.TableNamespaceManager;
66 import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
67 import org.apache.hadoop.hbase.quotas.QuotaExceededException;
68 import org.apache.hadoop.hbase.quotas.QuotaUtil;
69 import org.apache.hadoop.hbase.regionserver.HRegion;
70 import org.apache.hadoop.hbase.regionserver.HRegionServer;
71 import org.apache.hadoop.hbase.regionserver.Region;
72 import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost;
73 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
74 import org.apache.hadoop.hbase.testclassification.MediumTests;
75 import org.apache.hadoop.hbase.util.Bytes;
76 import org.apache.hadoop.hbase.util.FSUtils;
77 import org.apache.zookeeper.KeeperException;
78 import org.junit.After;
79 import org.junit.AfterClass;
80 import org.junit.BeforeClass;
81 import org.junit.Ignore;
82 import org.junit.Rule;
83 import org.junit.Test;
84 import org.junit.experimental.categories.Category;
85 import org.junit.rules.TestRule;
86
87 import com.google.common.collect.Sets;
88
89 @Category(MediumTests.class)
90 public class TestNamespaceAuditor {
91 @Rule public final TestRule timeout = CategoryBasedTimeout.builder().
92 withTimeout(this.getClass()).withLookingForStuckThread(true).build();
93 private static final Log LOG = LogFactory.getLog(TestNamespaceAuditor.class);
94 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
95 private static HBaseAdmin ADMIN;
96 private String prefix = "TestNamespaceAuditor";
97
98 @BeforeClass
99 public static void before() throws Exception {
100 UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
101 setupOnce();
102 }
103
104 public static void setupOnce() throws Exception, IOException {
105 Configuration conf = UTIL.getConfiguration();
106 conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName());
107 conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MasterSyncObserver.class.getName());
108 conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
109 conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
110 conf.setClass("hbase.coprocessor.regionserver.classes", CPRegionServerObserver.class,
111 RegionServerObserver.class);
112 UTIL.startMiniCluster(1, 1);
113 waitForQuotaEnabled();
114 ADMIN = UTIL.getHBaseAdmin();
115 }
116
117 @AfterClass
118 public static void tearDown() throws Exception {
119 UTIL.shutdownMiniCluster();
120 }
121
122 @After
123 public void cleanup() throws Exception, KeeperException {
124 for (HTableDescriptor table : ADMIN.listTables()) {
125 ADMIN.disableTable(table.getTableName());
126 deleteTable(table.getTableName());
127 }
128 for (NamespaceDescriptor ns : ADMIN.listNamespaceDescriptors()) {
129 if (ns.getName().startsWith(prefix)) {
130 ADMIN.deleteNamespace(ns.getName());
131 }
132 }
133 assertTrue("Quota manager not enabled", UTIL.getHBaseCluster().getMaster()
134 .getMasterQuotaManager().isQuotaEnabled());
135 }
136
137 @Test
138 public void testTableOperations() throws Exception {
139 String nsp = prefix + "_np2";
140 NamespaceDescriptor nspDesc =
141 NamespaceDescriptor.create(nsp)
142 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5")
143 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
144 ADMIN.createNamespace(nspDesc);
145 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
146 assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
147 HTableDescriptor tableDescOne =
148 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
149 HTableDescriptor tableDescTwo =
150 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
151 HTableDescriptor tableDescThree =
152 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table3"));
153 ADMIN.createTable(tableDescOne);
154 boolean constraintViolated = false;
155 try {
156 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
157 } catch (Exception exp) {
158 assertTrue(exp instanceof IOException);
159 constraintViolated = true;
160 } finally {
161 assertTrue("Constraint not violated for table " + tableDescTwo.getTableName(),
162 constraintViolated);
163 }
164 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
165 NamespaceTableAndRegionInfo nspState = getQuotaManager().getState(nsp);
166 assertNotNull(nspState);
167 assertTrue(nspState.getTables().size() == 2);
168 assertTrue(nspState.getRegionCount() == 5);
169 constraintViolated = false;
170 try {
171 ADMIN.createTable(tableDescThree);
172 } catch (Exception exp) {
173 assertTrue(exp instanceof IOException);
174 constraintViolated = true;
175 } finally {
176 assertTrue("Constraint not violated for table " + tableDescThree.getTableName(),
177 constraintViolated);
178 }
179 }
180
181 @Test
182 public void testValidQuotas() throws Exception {
183 boolean exceptionCaught = false;
184 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
185 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
186 NamespaceDescriptor nspDesc =
187 NamespaceDescriptor.create(prefix + "vq1")
188 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "hihdufh")
189 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
190 try {
191 ADMIN.createNamespace(nspDesc);
192 } catch (Exception exp) {
193 LOG.warn(exp);
194 exceptionCaught = true;
195 } finally {
196 assertTrue(exceptionCaught);
197 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
198 }
199 nspDesc =
200 NamespaceDescriptor.create(prefix + "vq2")
201 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "-456")
202 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
203 try {
204 ADMIN.createNamespace(nspDesc);
205 } catch (Exception exp) {
206 LOG.warn(exp);
207 exceptionCaught = true;
208 } finally {
209 assertTrue(exceptionCaught);
210 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
211 }
212 nspDesc =
213 NamespaceDescriptor.create(prefix + "vq3")
214 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
215 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "sciigd").build();
216 try {
217 ADMIN.createNamespace(nspDesc);
218 } catch (Exception exp) {
219 LOG.warn(exp);
220 exceptionCaught = true;
221 } finally {
222 assertTrue(exceptionCaught);
223 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
224 }
225 nspDesc =
226 NamespaceDescriptor.create(prefix + "vq4")
227 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
228 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "-1500").build();
229 try {
230 ADMIN.createNamespace(nspDesc);
231 } catch (Exception exp) {
232 LOG.warn(exp);
233 exceptionCaught = true;
234 } finally {
235 assertTrue(exceptionCaught);
236 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
237 }
238 }
239
240 @Test
241 public void testDeleteTable() throws Exception {
242 String namespace = prefix + "_dummy";
243 NamespaceDescriptor nspDesc =
244 NamespaceDescriptor.create(namespace)
245 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "100")
246 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "3").build();
247 ADMIN.createNamespace(nspDesc);
248 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(namespace));
249 NamespaceTableAndRegionInfo stateInfo = getNamespaceState(nspDesc.getName());
250 assertNotNull("Namespace state found null for " + namespace, stateInfo);
251 HTableDescriptor tableDescOne =
252 new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table1"));
253 HTableDescriptor tableDescTwo =
254 new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table2"));
255 ADMIN.createTable(tableDescOne);
256 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
257 stateInfo = getNamespaceState(nspDesc.getName());
258 assertNotNull("Namespace state found to be null.", stateInfo);
259 assertEquals(2, stateInfo.getTables().size());
260 assertEquals(5, stateInfo.getRegionCountOfTable(tableDescTwo.getTableName()));
261 assertEquals(6, stateInfo.getRegionCount());
262 ADMIN.disableTable(tableDescOne.getTableName());
263 deleteTable(tableDescOne.getTableName());
264 stateInfo = getNamespaceState(nspDesc.getName());
265 assertNotNull("Namespace state found to be null.", stateInfo);
266 assertEquals(5, stateInfo.getRegionCount());
267 assertEquals(1, stateInfo.getTables().size());
268 ADMIN.disableTable(tableDescTwo.getTableName());
269 deleteTable(tableDescTwo.getTableName());
270 ADMIN.deleteNamespace(namespace);
271 stateInfo = getNamespaceState(namespace);
272 assertNull("Namespace state not found to be null.", stateInfo);
273 }
274
275 public static class CPRegionServerObserver extends BaseRegionServerObserver {
276 private volatile boolean shouldFailMerge = false;
277
278 public void failMerge(boolean fail) {
279 shouldFailMerge = fail;
280 }
281
282 private boolean triggered = false;
283
284 public synchronized void waitUtilTriggered() throws InterruptedException {
285 while (!triggered) {
286 wait();
287 }
288 }
289
290 @Override
291 public synchronized void preMerge(ObserverContext<RegionServerCoprocessorEnvironment> ctx,
292 Region regionA, Region regionB) throws IOException {
293 triggered = true;
294 notifyAll();
295 if (shouldFailMerge) {
296 throw new IOException("fail merge");
297 }
298 }
299 }
300
301 @Test
302 public void testRegionMerge() throws Exception {
303 String nsp1 = prefix + "_regiontest";
304 NamespaceDescriptor nspDesc =
305 NamespaceDescriptor.create(nsp1)
306 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3")
307 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
308 ADMIN.createNamespace(nspDesc);
309 final TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
310 byte[] columnFamily = Bytes.toBytes("info");
311 HTableDescriptor tableDescOne = new HTableDescriptor(tableTwo);
312 tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
313 final int initialRegions = 3;
314 ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("2000"), initialRegions);
315 try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
316 Table table = connection.getTable(tableTwo)) {
317 UTIL.loadNumericRows(table, Bytes.toBytes("info"), 1000, 1999);
318 }
319 ADMIN.flush(tableTwo);
320 List<HRegionInfo> hris = ADMIN.getTableRegions(tableTwo);
321 Collections.sort(hris);
322
323 final Set<String> encodedRegionNamesToMerge =
324 Sets.newHashSet(hris.get(0).getEncodedName(), hris.get(1).getEncodedName());
325 ADMIN.mergeRegions(hris.get(0).getEncodedNameAsBytes(), hris.get(1).getEncodedNameAsBytes(),
326 false);
327 waitForMergeToComplete(tableTwo, encodedRegionNamesToMerge);
328 hris = ADMIN.getTableRegions(tableTwo);
329 assertEquals(initialRegions - 1, hris.size());
330 Collections.sort(hris);
331
332 final HRegionInfo hriToSplit = hris.get(1);
333 ADMIN.split(tableTwo, Bytes.toBytes("500"));
334
335 UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate<Exception>() {
336
337 @Override
338 public boolean evaluate() throws Exception {
339 RegionStates regionStates =
340 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
341 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
342 if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
343 return false;
344 }
345 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
346 return false;
347 }
348 }
349 return true;
350 }
351
352 @Override
353 public String explainFailure() throws Exception {
354 RegionStates regionStates =
355 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
356 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
357 if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
358 return hriToSplit + " which is expected to be split is still online";
359 }
360 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
361 return hri + " is still in not opened";
362 }
363 }
364 return "Unknown";
365 }
366 });
367 hris = ADMIN.getTableRegions(tableTwo);
368 assertEquals(initialRegions, hris.size());
369 Collections.sort(hris);
370
371
372 MiniHBaseCluster cluster = UTIL.getHBaseCluster();
373 HRegionServer regionServer = cluster.getRegionServer(0);
374 RegionServerCoprocessorHost cpHost = regionServer.getRegionServerCoprocessorHost();
375 Coprocessor coprocessor = cpHost.findCoprocessor(CPRegionServerObserver.class.getName());
376 CPRegionServerObserver regionServerObserver = (CPRegionServerObserver) coprocessor;
377 regionServerObserver.failMerge(true);
378 regionServerObserver.triggered = false;
379
380 ADMIN.mergeRegions(hris.get(1).getEncodedNameAsBytes(), hris.get(2).getEncodedNameAsBytes(),
381 false);
382 regionServerObserver.waitUtilTriggered();
383 hris = ADMIN.getTableRegions(tableTwo);
384 assertEquals(initialRegions, hris.size());
385 Collections.sort(hris);
386
387 HRegionInfo hriToSplit2 = hris.get(1);
388 ADMIN.split(tableTwo,
389 TableInputFormatBase.getSplitKey(hriToSplit2.getStartKey(), hriToSplit2.getEndKey(), true));
390 waitForMergeToComplete(tableTwo, encodedRegionNamesToMerge);
391 assertEquals(initialRegions, ADMIN.getTableRegions(tableTwo).size());
392 }
393
394 private void waitForMergeToComplete(final TableName tableTwo,
395 final Set<String> encodedRegionNamesToMerge) throws Exception {
396 UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate<Exception>() {
397
398 @Override
399 public boolean evaluate() throws Exception {
400 RegionStates regionStates =
401 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
402 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
403 if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
404 return false;
405 }
406 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
407 return false;
408 }
409 }
410 return true;
411 }
412
413 @Override
414 public String explainFailure() throws Exception {
415 RegionStates regionStates =
416 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
417 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
418 if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
419 return hri + " which is expected to be merged is still online";
420 }
421 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
422 return hri + " is still in not opened";
423 }
424 }
425 return "Unknown";
426 }
427 });
428 }
429
430 @Ignore("Hangs on occasion waiting on countdown latch") @Test
431 public void testRegionOperations() throws Exception {
432 String nsp1 = prefix + "_regiontest";
433 NamespaceDescriptor nspDesc =
434 NamespaceDescriptor.create(nsp1)
435 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2")
436 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
437 ADMIN.createNamespace(nspDesc);
438 boolean constraintViolated = false;
439 final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
440 byte[] columnFamily = Bytes.toBytes("info");
441 HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
442 tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
443 NamespaceTableAndRegionInfo stateInfo;
444 try {
445 ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 7);
446 } catch (Exception exp) {
447 assertTrue(exp instanceof DoNotRetryIOException);
448 LOG.info(exp);
449 constraintViolated = true;
450 } finally {
451 assertTrue(constraintViolated);
452 }
453 assertFalse(ADMIN.tableExists(tableOne));
454
455 ADMIN.createTable(tableDescOne);
456 Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
457 HTable htable = (HTable) connection.getTable(tableOne);
458 UTIL.loadNumericRows(htable, Bytes.toBytes("info"), 1, 1000);
459 ADMIN.flush(tableOne);
460 stateInfo = getNamespaceState(nsp1);
461 assertEquals(1, stateInfo.getTables().size());
462 assertEquals(1, stateInfo.getRegionCount());
463 restartMaster();
464 ADMIN.split(tableOne, Bytes.toBytes("500"));
465 HRegion actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
466 CustomObserver observer =
467 (CustomObserver) actualRegion.getCoprocessorHost().findCoprocessor(
468 CustomObserver.class.getName());
469 assertNotNull(observer);
470 observer.postSplit.await();
471 assertEquals(2, ADMIN.getTableRegions(tableOne).size());
472 actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
473 observer =
474 (CustomObserver) actualRegion.getCoprocessorHost().findCoprocessor(
475 CustomObserver.class.getName());
476 assertNotNull(observer);
477 ADMIN.split(
478 tableOne,
479 getSplitKey(actualRegion.getRegionInfo().getStartKey(), actualRegion.getRegionInfo()
480 .getEndKey()));
481 observer.postSplit.await();
482
483 List<HRegionInfo> hris = ADMIN.getTableRegions(tableOne);
484 assertEquals(2, hris.size());
485 assertTrue("split completed", observer.preSplitBeforePONR.getCount() == 1);
486
487 htable.close();
488 }
489
490
491
492
493
494
495 @Test
496 public void testRecreateTableWithSameNameAfterFirstTimeFailure() throws Exception {
497 String nsp1 = prefix + "_testRecreateTable";
498 NamespaceDescriptor nspDesc =
499 NamespaceDescriptor.create(nsp1)
500 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
501 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1").build();
502 ADMIN.createNamespace(nspDesc);
503 final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
504 byte[] columnFamily = Bytes.toBytes("info");
505 HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
506 tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
507 MasterSyncObserver.throwExceptionInPreCreateTableHandler = true;
508 try {
509 try {
510 ADMIN.createTable(tableDescOne);
511 fail("Table " + tableOne.toString() + "creation should fail.");
512 } catch (Exception exp) {
513 LOG.error(exp);
514 }
515 assertFalse(ADMIN.tableExists(tableOne));
516
517 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp1);
518 assertEquals("First table creation failed in namespace so number of tables in namespace "
519 + "should be 0.", 0, nstate.getTables().size());
520
521 MasterSyncObserver.throwExceptionInPreCreateTableHandler = false;
522 try {
523 ADMIN.createTable(tableDescOne);
524 } catch (Exception e) {
525 fail("Table " + tableOne.toString() + "creation should succeed.");
526 LOG.error(e);
527 }
528 assertTrue(ADMIN.tableExists(tableOne));
529 nstate = getNamespaceState(nsp1);
530 assertEquals("First table was created successfully so table size in namespace should "
531 + "be one now.", 1, nstate.getTables().size());
532 } finally {
533 MasterSyncObserver.throwExceptionInPreCreateTableHandler = false;
534 if (ADMIN.tableExists(tableOne)) {
535 ADMIN.disableTable(tableOne);
536 deleteTable(tableOne);
537 }
538 ADMIN.deleteNamespace(nsp1);
539 }
540 }
541
542 private NamespaceTableAndRegionInfo getNamespaceState(String namespace) throws KeeperException,
543 IOException {
544 return getQuotaManager().getState(namespace);
545 }
546
547 byte[] getSplitKey(byte[] startKey, byte[] endKey) {
548 String skey = Bytes.toString(startKey);
549 int key;
550 if (StringUtils.isBlank(skey)) {
551 key = Integer.parseInt(Bytes.toString(endKey)) / 2;
552 } else {
553 key = (int) (Integer.parseInt(skey) * 1.5);
554 }
555 return Bytes.toBytes("" + key);
556 }
557
558 public static class CustomObserver extends BaseRegionObserver {
559 volatile CountDownLatch postSplit;
560 volatile CountDownLatch preSplitBeforePONR;
561
562 @Override
563 public void postCompleteSplit(ObserverContext<RegionCoprocessorEnvironment> ctx)
564 throws IOException {
565 postSplit.countDown();
566 }
567
568 @Override
569 public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
570 byte[] splitKey, List<Mutation> metaEntries) throws IOException {
571 preSplitBeforePONR.countDown();
572 }
573
574 @Override
575 public void start(CoprocessorEnvironment e) throws IOException {
576 postSplit = new CountDownLatch(1);
577 preSplitBeforePONR = new CountDownLatch(1);
578 }
579 }
580
581 @Test
582 public void testStatePreserve() throws Exception {
583 final String nsp1 = prefix + "_testStatePreserve";
584 NamespaceDescriptor nspDesc =
585 NamespaceDescriptor.create(nsp1)
586 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
587 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "10").build();
588 ADMIN.createNamespace(nspDesc);
589 TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
590 TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
591 TableName tableThree = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table3");
592 HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
593 HTableDescriptor tableDescTwo = new HTableDescriptor(tableTwo);
594 HTableDescriptor tableDescThree = new HTableDescriptor(tableThree);
595 ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
596 ADMIN.createTable(tableDescTwo, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
597 ADMIN.createTable(tableDescThree, Bytes.toBytes("1"), Bytes.toBytes("1000"), 4);
598 ADMIN.disableTable(tableThree);
599 deleteTable(tableThree);
600
601 UTIL.waitFor(1000, new Waiter.Predicate<Exception>() {
602 @Override
603 public boolean evaluate() throws Exception {
604 return (getNamespaceState(nsp1).getTables().size() == 2);
605 }
606 });
607 NamespaceTableAndRegionInfo before = getNamespaceState(nsp1);
608 restartMaster();
609 NamespaceTableAndRegionInfo after = getNamespaceState(nsp1);
610 assertEquals("Expected: " + before.getTables() + " Found: " + after.getTables(), before
611 .getTables().size(), after.getTables().size());
612 }
613
614 private static void waitForQuotaEnabled() throws Exception {
615 UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
616 @Override
617 public boolean evaluate() throws Exception {
618 HMaster master = UTIL.getHBaseCluster().getMaster();
619 if (master == null) {
620 return false;
621 }
622 MasterQuotaManager quotaManager = master.getMasterQuotaManager();
623 return quotaManager != null && quotaManager.isQuotaEnabled();
624 }
625 });
626 }
627
628 private void restartMaster() throws Exception {
629 UTIL.getHBaseCluster().getMaster(0).stop("Stopping to start again");
630 UTIL.getHBaseCluster().waitOnMaster(0);
631 UTIL.getHBaseCluster().startMaster();
632 waitForQuotaEnabled();
633 }
634
635 private NamespaceAuditor getQuotaManager() {
636 return UTIL.getHBaseCluster().getMaster().getMasterQuotaManager().getNamespaceQuotaManager();
637 }
638
639 public static class MasterSyncObserver extends BaseMasterObserver {
640 volatile CountDownLatch tableDeletionLatch;
641 static boolean throwExceptionInPreCreateTableHandler;
642
643 @Override
644 public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
645 TableName tableName) throws IOException {
646 tableDeletionLatch = new CountDownLatch(1);
647 }
648
649 @Override
650 public void postDeleteTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
651 TableName tableName) throws IOException {
652 tableDeletionLatch.countDown();
653 }
654
655 @Override
656 public void preCreateTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
657 HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
658 if (throwExceptionInPreCreateTableHandler) {
659 throw new IOException("Throw exception as it is demanded.");
660 }
661 }
662 }
663
664 private void deleteTable(final TableName tableName) throws Exception {
665
666
667 MasterSyncObserver observer =
668 (MasterSyncObserver) UTIL.getHBaseCluster().getMaster().getMasterCoprocessorHost()
669 .findCoprocessor(MasterSyncObserver.class.getName());
670 ADMIN.deleteTable(tableName);
671 observer.tableDeletionLatch.await();
672 }
673
674 @Test(expected = QuotaExceededException.class)
675 public void testExceedTableQuotaInNamespace() throws Exception {
676 String nsp = prefix + "_testExceedTableQuotaInNamespace";
677 NamespaceDescriptor nspDesc =
678 NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
679 .build();
680 ADMIN.createNamespace(nspDesc);
681 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
682 assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
683 HTableDescriptor tableDescOne =
684 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
685 HTableDescriptor tableDescTwo =
686 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
687 ADMIN.createTable(tableDescOne);
688 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
689 }
690
691 @Test(expected = QuotaExceededException.class)
692 public void testCloneSnapshotQuotaExceed() throws Exception {
693 String nsp = prefix + "_testTableQuotaExceedWithCloneSnapshot";
694 NamespaceDescriptor nspDesc =
695 NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
696 .build();
697 ADMIN.createNamespace(nspDesc);
698 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
699 TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
700 TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
701 HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
702 ADMIN.createTable(tableDescOne);
703 String snapshot = "snapshot_testTableQuotaExceedWithCloneSnapshot";
704 ADMIN.snapshot(snapshot, tableName);
705 ADMIN.cloneSnapshot(snapshot, cloneTableName);
706 ADMIN.deleteSnapshot(snapshot);
707 }
708
709 @Test
710 public void testCloneSnapshot() throws Exception {
711 String nsp = prefix + "_testCloneSnapshot";
712 NamespaceDescriptor nspDesc =
713 NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
714 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20").build();
715 ADMIN.createNamespace(nspDesc);
716 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
717 TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
718 TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
719 HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
720
721 ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
722 String snapshot = "snapshot_testCloneSnapshot";
723 ADMIN.snapshot(snapshot, tableName);
724 ADMIN.cloneSnapshot(snapshot, cloneTableName);
725
726 int tableLength;
727 try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(tableName)) {
728 tableLength = locator.getStartKeys().length;
729 }
730 assertEquals(tableName.getNameAsString() + " should have four regions.", 4, tableLength);
731
732 try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(cloneTableName)) {
733 tableLength = locator.getStartKeys().length;
734 }
735 assertEquals(cloneTableName.getNameAsString() + " should have four regions.", 4, tableLength);
736
737 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
738 assertEquals("Total tables count should be 2.", 2, nstate.getTables().size());
739 assertEquals("Total regions count should be.", 8, nstate.getRegionCount());
740
741 ADMIN.deleteSnapshot(snapshot);
742 }
743
744 @Test
745 public void testRestoreSnapshot() throws Exception {
746 String nsp = prefix + "_testRestoreSnapshot";
747 NamespaceDescriptor nspDesc =
748 NamespaceDescriptor.create(nsp)
749 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
750 ADMIN.createNamespace(nspDesc);
751 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
752 TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
753 HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
754 ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
755
756 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
757 assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
758
759 String snapshot = "snapshot_testRestoreSnapshot";
760 ADMIN.snapshot(snapshot, tableName1);
761
762 List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
763 Collections.sort(regions);
764
765 ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
766 Thread.sleep(2000);
767 assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
768
769 ADMIN.disableTable(tableName1);
770 ADMIN.restoreSnapshot(snapshot);
771
772 assertEquals("Total regions count should be 4 after restore.", 4, nstate.getRegionCount());
773
774 ADMIN.enableTable(tableName1);
775 ADMIN.deleteSnapshot(snapshot);
776 }
777
778 @Test
779 public void testRestoreSnapshotQuotaExceed() throws Exception {
780 String nsp = prefix + "_testRestoreSnapshotQuotaExceed";
781 NamespaceDescriptor nspDesc =
782 NamespaceDescriptor.create(nsp)
783 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
784 ADMIN.createNamespace(nspDesc);
785 NamespaceDescriptor ndesc = ADMIN.getNamespaceDescriptor(nsp);
786 assertNotNull("Namespace descriptor found null.", ndesc);
787 TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
788 HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
789 ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
790
791 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
792 assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
793
794 String snapshot = "snapshot_testRestoreSnapshotQuotaExceed";
795
796 ADMIN.snapshot(snapshot, tableName1);
797
798 ADMIN.disableTable(tableName1);
799 ADMIN.deleteTable(tableName1);
800 ADMIN.createTable(tableDescOne);
801 ndesc.setConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3");
802 ADMIN.modifyNamespace(ndesc);
803
804 ADMIN.disableTable(tableName1);
805 try {
806 ADMIN.restoreSnapshot(snapshot);
807 fail("Region quota is exceeded so QuotaExceededException should be thrown but HBaseAdmin"
808 + " wraps IOException into RestoreSnapshotException");
809 } catch (RestoreSnapshotException ignore) {
810 assertTrue(ignore.getCause() instanceof QuotaExceededException);
811 }
812 assertEquals(1, getNamespaceState(nsp).getRegionCount());
813 ADMIN.enableTable(tableName1);
814 ADMIN.deleteSnapshot(snapshot);
815 }
816 }