View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  import static org.junit.Assert.fail;
27  
28  import java.io.IOException;
29  import java.util.Set;
30  import java.util.concurrent.Callable;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.fs.FileSystem;
35  import org.apache.hadoop.fs.Path;
36  import org.apache.hadoop.hbase.client.Admin;
37  import org.apache.hadoop.hbase.client.Get;
38  import org.apache.hadoop.hbase.client.HTable;
39  import org.apache.hadoop.hbase.client.Put;
40  import org.apache.hadoop.hbase.client.Table;
41  import org.apache.hadoop.hbase.master.HMaster;
42  import org.apache.hadoop.hbase.testclassification.MediumTests;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.apache.hadoop.hbase.util.FSUtils;
45  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
46  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
47  import org.junit.AfterClass;
48  import org.junit.Assert;
49  import org.junit.Before;
50  import org.junit.BeforeClass;
51  import org.junit.Ignore;
52  import org.junit.Test;
53  import org.junit.experimental.categories.Category;
54  
55  import com.google.common.collect.Sets;
56  
57  @Category(MediumTests.class)
58  public class TestNamespace {
59    private static final Log LOG = LogFactory.getLog(TestNamespace.class);
60    private static HMaster master;
61    protected final static int NUM_SLAVES_BASE = 4;
62    private static HBaseTestingUtility TEST_UTIL;
63    protected static Admin admin;
64    protected static HBaseCluster cluster;
65    private static ZKNamespaceManager zkNamespaceManager;
66    private String prefix = "TestNamespace";
67  
68  
69    @BeforeClass
70    public static void setUp() throws Exception {
71      TEST_UTIL = new HBaseTestingUtility();
72      TEST_UTIL.getConfiguration().setInt("hbase.namespacejanitor.interval", 5000);
73      TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE);
74      admin = TEST_UTIL.getHBaseAdmin();
75      cluster = TEST_UTIL.getHBaseCluster();
76      master = ((MiniHBaseCluster)cluster).getMaster();
77      zkNamespaceManager =
78          new ZKNamespaceManager(master.getZooKeeper());
79      zkNamespaceManager.start();
80      LOG.info("Done initializing cluster");
81    }
82  
83    @AfterClass
84    public static void tearDown() throws Exception {
85      TEST_UTIL.shutdownMiniCluster();
86    }
87  
88    @Before
89    public void beforeMethod() throws IOException {
90      for (HTableDescriptor desc : admin.listTables(prefix+".*")) {
91        admin.disableTable(desc.getTableName());
92        admin.deleteTable(desc.getTableName());
93      }
94      for (NamespaceDescriptor ns : admin.listNamespaceDescriptors()) {
95        if (ns.getName().startsWith(prefix)) {
96          admin.deleteNamespace(ns.getName());
97        }
98      }
99    }
100 
101   @Test
102   public void verifyReservedNS() throws IOException {
103     //verify existence of reserved namespaces
104     NamespaceDescriptor ns =
105         admin.getNamespaceDescriptor(NamespaceDescriptor.DEFAULT_NAMESPACE.getName());
106     assertNotNull(ns);
107     assertEquals(ns.getName(), NamespaceDescriptor.DEFAULT_NAMESPACE.getName());
108     assertNotNull(zkNamespaceManager.get(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR));
109 
110     ns = admin.getNamespaceDescriptor(NamespaceDescriptor.SYSTEM_NAMESPACE.getName());
111     assertNotNull(ns);
112     assertEquals(ns.getName(), NamespaceDescriptor.SYSTEM_NAMESPACE.getName());
113     assertNotNull(zkNamespaceManager.get(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR));
114 
115     assertEquals(2, admin.listNamespaceDescriptors().length);
116 
117     //verify existence of system tables
118     Set<TableName> systemTables = Sets.newHashSet(
119         TableName.META_TABLE_NAME,
120         TableName.NAMESPACE_TABLE_NAME);
121     HTableDescriptor[] descs =
122         admin.listTableDescriptorsByNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE.getName());
123     assertEquals(systemTables.size(), descs.length);
124     for (HTableDescriptor desc : descs) {
125       assertTrue(systemTables.contains(desc.getTableName()));
126     }
127     //verify system tables aren't listed
128     assertEquals(0, admin.listTables().length);
129 
130     //Try creating default and system namespaces.
131     boolean exceptionCaught = false;
132     try {
133       admin.createNamespace(NamespaceDescriptor.DEFAULT_NAMESPACE);
134     } catch (IOException exp) {
135       LOG.warn(exp);
136       exceptionCaught = true;
137     } finally {
138       assertTrue(exceptionCaught);
139     }
140 
141     exceptionCaught = false;
142     try {
143       admin.createNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE);
144     } catch (IOException exp) {
145       LOG.warn(exp);
146       exceptionCaught = true;
147     } finally {
148       assertTrue(exceptionCaught);
149     }
150   }
151 
152   @Test
153   public void testDeleteReservedNS() throws Exception {
154     boolean exceptionCaught = false;
155     try {
156       admin.deleteNamespace(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR);
157     } catch (IOException exp) {
158       LOG.warn(exp);
159       exceptionCaught = true;
160     } finally {
161       assertTrue(exceptionCaught);
162     }
163 
164     try {
165       admin.deleteNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR);
166     } catch (IOException exp) {
167       LOG.warn(exp);
168       exceptionCaught = true;
169     } finally {
170       assertTrue(exceptionCaught);
171     }
172   }
173 
174   @Test
175   public void createRemoveTest() throws Exception {
176     String testName = "createRemoveTest";
177     String nsName = prefix+"_"+testName;
178     LOG.info(testName);
179 
180     //create namespace and verify
181     admin.createNamespace(NamespaceDescriptor.create(nsName).build());
182     assertEquals(3, admin.listNamespaceDescriptors().length);
183     TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
184       @Override
185       public boolean evaluate() throws Exception {
186         return zkNamespaceManager.list().size() == 3;
187       }
188     });
189     assertNotNull(zkNamespaceManager.get(nsName));
190     //remove namespace and verify
191     admin.deleteNamespace(nsName);
192     assertEquals(2, admin.listNamespaceDescriptors().length);
193     assertEquals(2, zkNamespaceManager.list().size());
194     assertNull(zkNamespaceManager.get(nsName));
195   }
196 
197   @Test
198   public void createDoubleTest() throws IOException, InterruptedException {
199     String testName = "createDoubleTest";
200     String nsName = prefix+"_"+testName;
201     LOG.info(testName);
202 
203     TableName tableName = TableName.valueOf("my_table");
204     TableName tableNameFoo = TableName.valueOf(nsName+":my_table");
205     //create namespace and verify
206     admin.createNamespace(NamespaceDescriptor.create(nsName).build());
207     TEST_UTIL.createTable(tableName, Bytes.toBytes(nsName));
208     TEST_UTIL.createTable(tableNameFoo,Bytes.toBytes(nsName));
209     assertEquals(2, admin.listTables().length);
210     assertNotNull(admin
211         .getTableDescriptor(tableName));
212     assertNotNull(admin
213         .getTableDescriptor(tableNameFoo));
214     //remove namespace and verify
215     admin.disableTable(tableName);
216     admin.deleteTable(tableName);
217     assertEquals(1, admin.listTables().length);
218   }
219 
220   @Test
221   public void createTableTest() throws IOException, InterruptedException {
222     String testName = "createTableTest";
223     String nsName = prefix+"_"+testName;
224     LOG.info(testName);
225 
226     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(nsName+":my_table"));
227     HColumnDescriptor colDesc = new HColumnDescriptor("my_cf");
228     desc.addFamily(colDesc);
229     try {
230       admin.createTable(desc);
231       fail("Expected no namespace exists exception");
232     } catch (NamespaceNotFoundException ex) {
233     }
234     //create table and in new namespace
235     admin.createNamespace(NamespaceDescriptor.create(nsName).build());
236     admin.createTable(desc);
237     TEST_UTIL.waitTableAvailable(desc.getTableName().getName(), 10000);
238     FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
239     assertTrue(fs.exists(
240         new Path(master.getMasterFileSystem().getRootDir(),
241             new Path(HConstants.BASE_NAMESPACE_DIR,
242                 new Path(nsName, desc.getTableName().getQualifierAsString())))));
243     assertEquals(1, admin.listTables().length);
244 
245     //verify non-empty namespace can't be removed
246     try {
247       admin.deleteNamespace(nsName);
248       fail("Expected non-empty namespace constraint exception");
249     } catch (Exception ex) {
250       LOG.info("Caught expected exception: " + ex);
251     }
252 
253     //sanity check try to write and read from table
254     Table table = new HTable(TEST_UTIL.getConfiguration(), desc.getTableName());
255     Put p = new Put(Bytes.toBytes("row1"));
256     p.add(Bytes.toBytes("my_cf"),Bytes.toBytes("my_col"),Bytes.toBytes("value1"));
257     table.put(p);
258     //flush and read from disk to make sure directory changes are working
259     admin.flush(desc.getTableName());
260     Get g = new Get(Bytes.toBytes("row1"));
261     assertTrue(table.exists(g));
262 
263     //normal case of removing namespace
264     TEST_UTIL.deleteTable(desc.getTableName());
265     admin.deleteNamespace(nsName);
266   }
267 
268   @Test
269   public void createTableInDefaultNamespace() throws Exception {
270     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf("default_table"));
271     HColumnDescriptor colDesc = new HColumnDescriptor("cf1");
272     desc.addFamily(colDesc);
273     admin.createTable(desc);
274     assertTrue(admin.listTables().length == 1);
275     admin.disableTable(desc.getTableName());
276     admin.deleteTable(desc.getTableName());
277   }
278 
279   @Test
280   public void createTableInSystemNamespace() throws Exception {
281     TableName tableName = TableName.valueOf("hbase:createTableInSystemNamespace");
282     HTableDescriptor desc = new HTableDescriptor(tableName);
283     HColumnDescriptor colDesc = new HColumnDescriptor("cf1");
284     desc.addFamily(colDesc);
285     admin.createTable(desc);
286     assertEquals(0, admin.listTables().length);
287     assertTrue(admin.tableExists(tableName));
288     admin.disableTable(desc.getTableName());
289     admin.deleteTable(desc.getTableName());
290   }
291 
292   @Ignore @Test
293   public void testNamespaceJanitor() throws Exception {
294     FileSystem fs = TEST_UTIL.getTestFileSystem();
295 
296     int fsCount = fs.listStatus(new Path(FSUtils.getRootDir(TEST_UTIL.getConfiguration()),
297         HConstants.BASE_NAMESPACE_DIR)).length;
298     Path fakeNSPath =
299         FSUtils.getNamespaceDir(FSUtils.getRootDir(TEST_UTIL.getConfiguration()), "foo");
300     assertTrue(fs.mkdirs(fakeNSPath));
301 
302     String fakeZnode = ZKUtil.joinZNode(ZooKeeperWatcher.namespaceZNode, "foo");
303     int zkCount = ZKUtil.listChildrenNoWatch(TEST_UTIL.getZooKeeperWatcher(),
304         ZooKeeperWatcher.namespaceZNode).size();
305     ZKUtil.createWithParents(TEST_UTIL.getZooKeeperWatcher(), fakeZnode);
306     Thread.sleep(10000);
307 
308     //verify namespace count is the same and orphan is removed
309     assertFalse(fs.exists(fakeNSPath));
310     assertEquals(fsCount, fs.listStatus(new Path(FSUtils.getRootDir(TEST_UTIL.getConfiguration()),
311             HConstants.BASE_NAMESPACE_DIR)).length);
312 
313     assertEquals(-1, ZKUtil.checkExists(TEST_UTIL.getZooKeeperWatcher(), fakeZnode));
314     assertEquals(zkCount,
315         ZKUtil.listChildrenNoWatch(TEST_UTIL.getZooKeeperWatcher(),
316             ZooKeeperWatcher.namespaceZNode).size());
317   }
318 
319   @Test(timeout = 60000)
320   public void testNamespaceOperations() throws IOException {
321     admin.createNamespace(NamespaceDescriptor.create(prefix + "ns1").build());
322     admin.createNamespace(NamespaceDescriptor.create(prefix + "ns2").build());
323 
324     // create namespace that already exists
325     runWithExpectedException(new Callable<Void>() {
326       @Override
327       public Void call() throws Exception {
328         admin.createNamespace(NamespaceDescriptor.create(prefix + "ns1").build());
329         return null;
330       }
331     }, NamespaceExistException.class);
332 
333     // create a table in non-existing namespace
334     runWithExpectedException(new Callable<Void>() {
335       @Override
336       public Void call() throws Exception {
337         HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("non_existing_namespace", "table1"));
338         htd.addFamily(new HColumnDescriptor("family1"));
339         admin.createTable(htd);
340         return null;
341       }
342     }, NamespaceNotFoundException.class);
343 
344     // get descriptor for existing namespace
345     admin.getNamespaceDescriptor(prefix + "ns1");
346 
347     // get descriptor for non-existing namespace
348     runWithExpectedException(new Callable<NamespaceDescriptor>() {
349       @Override
350       public NamespaceDescriptor call() throws Exception {
351         return admin.getNamespaceDescriptor("non_existing_namespace");
352       }
353     }, NamespaceNotFoundException.class);
354 
355     // delete descriptor for existing namespace
356     admin.deleteNamespace(prefix + "ns2");
357 
358     // delete descriptor for non-existing namespace
359     runWithExpectedException(new Callable<Void>() {
360       @Override
361       public Void call() throws Exception {
362         admin.deleteNamespace("non_existing_namespace");
363         return null;
364       }
365     }, NamespaceNotFoundException.class);
366 
367     // modify namespace descriptor for existing namespace
368     NamespaceDescriptor ns1 = admin.getNamespaceDescriptor(prefix + "ns1");
369     ns1.setConfiguration("foo", "bar");
370     admin.modifyNamespace(ns1);
371 
372     // modify namespace descriptor for non-existing namespace
373     runWithExpectedException(new Callable<Void>() {
374       @Override
375       public Void call() throws Exception {
376         admin.modifyNamespace(NamespaceDescriptor.create("non_existing_namespace").build());
377         return null;
378       }
379     }, NamespaceNotFoundException.class);
380 
381     // get table descriptors for existing namespace
382     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(prefix + "ns1", "table1"));
383     htd.addFamily(new HColumnDescriptor("family1"));
384     admin.createTable(htd);
385     HTableDescriptor[] htds = admin.listTableDescriptorsByNamespace(prefix + "ns1");
386     assertNotNull("Should have not returned null", htds);
387     assertEquals("Should have returned non-empty array", 1, htds.length);
388 
389     // get table descriptors for non-existing namespace
390     runWithExpectedException(new Callable<Void>() {
391       @Override
392       public Void call() throws Exception {
393         admin.listTableDescriptorsByNamespace("non_existing_namespace");
394         return null;
395       }
396     }, NamespaceNotFoundException.class);
397 
398     // get table names for existing namespace
399     TableName[] tableNames = admin.listTableNamesByNamespace(prefix + "ns1");
400     assertNotNull("Should have not returned null", tableNames);
401     assertEquals("Should have returned non-empty array", 1, tableNames.length);
402 
403     // get table names for non-existing namespace
404     runWithExpectedException(new Callable<Void>() {
405       @Override
406       public Void call() throws Exception {
407         admin.listTableNamesByNamespace("non_existing_namespace");
408         return null;
409       }
410     }, NamespaceNotFoundException.class);
411 
412   }
413 
414   private static <V, E> void runWithExpectedException(Callable<V> callable, Class<E> exceptionClass) {
415     try {
416       callable.call();
417     } catch(Exception ex) {
418       Assert.assertEquals(exceptionClass, ex.getClass());
419       return;
420     }
421     fail("Should have thrown exception " + exceptionClass);
422   }
423 
424 }