1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.migration;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.assertFalse;
25
26 import java.io.File;
27 import java.io.IOException;
28 import java.util.ArrayList;
29 import java.util.List;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.fs.FileSystem;
35 import org.apache.hadoop.fs.FileUtil;
36 import org.apache.hadoop.fs.FsShell;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.hbase.HBaseTestingUtility;
39 import org.apache.hadoop.hbase.HColumnDescriptor;
40 import org.apache.hadoop.hbase.HConstants;
41 import org.apache.hadoop.hbase.HRegionInfo;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.testclassification.MediumTests;
44 import org.apache.hadoop.hbase.NamespaceDescriptor;
45 import org.apache.hadoop.hbase.TableName;
46 import org.apache.hadoop.hbase.Waiter;
47 import org.apache.hadoop.hbase.client.Get;
48 import org.apache.hadoop.hbase.client.HTable;
49 import org.apache.hadoop.hbase.client.Put;
50 import org.apache.hadoop.hbase.client.Result;
51 import org.apache.hadoop.hbase.client.ResultScanner;
52 import org.apache.hadoop.hbase.client.Scan;
53 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
54 import org.apache.hadoop.hbase.regionserver.HRegion;
55 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
56 import org.apache.hadoop.hbase.security.access.AccessControlLists;
57 import org.apache.hadoop.hbase.util.Bytes;
58 import org.apache.hadoop.hbase.util.FSTableDescriptors;
59 import org.apache.hadoop.hbase.util.FSUtils;
60 import org.apache.hadoop.util.ToolRunner;
61 import org.junit.AfterClass;
62 import org.junit.Assert;
63 import org.junit.BeforeClass;
64 import org.junit.Test;
65 import org.junit.experimental.categories.Category;
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84 @Category(MediumTests.class)
85 public class TestNamespaceUpgrade {
86 static final Log LOG = LogFactory.getLog(TestNamespaceUpgrade.class);
87 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
88 private final static String snapshot1Keys[] =
89 {"1","10","2","3","4","5","6","7","8","9"};
90 private final static String snapshot2Keys[] =
91 {"1","2","3","4","5","6","7","8","9"};
92 private final static String currentKeys[] =
93 {"1","2","3","4","5","6","7","8","9","A"};
94 private final static TableName tables[] =
95 {TableName.valueOf("data"), TableName.valueOf("foo"),
96 TableName.valueOf("ns1.foo"), TableName.valueOf("ns.two.foo")};
97
98 @BeforeClass
99 public static void setUpBeforeClass() throws Exception {
100
101
102 TEST_UTIL.startMiniZKCluster();
103 TEST_UTIL.startMiniDFSCluster(1);
104 Path testdir = TEST_UTIL.getDataTestDir("TestNamespaceUpgrade");
105
106 File untar = untar(new File(testdir.toString()));
107
108 Configuration conf = TEST_UTIL.getConfiguration();
109 FsShell shell = new FsShell(conf);
110 FileSystem fs = FileSystem.get(conf);
111
112 Path hbaseRootDir = TEST_UTIL.getDefaultRootDirPath();
113 if (!fs.isDirectory(hbaseRootDir.getParent())) {
114
115 fs.mkdirs(hbaseRootDir.getParent());
116 }
117 if(org.apache.hadoop.util.VersionInfo.getVersion().startsWith("2.")) {
118 LOG.info("Hadoop version is 2.x, pre-migrating snapshot dir");
119 FileSystem localFS = FileSystem.getLocal(conf);
120 if(!localFS.rename(new Path(untar.toString(), HConstants.OLD_SNAPSHOT_DIR_NAME),
121 new Path(untar.toString(), HConstants.SNAPSHOT_DIR_NAME))) {
122 throw new IllegalStateException("Failed to move snapshot dir to 2.x expectation");
123 }
124 }
125 doFsCommand(shell,
126 new String [] {"-put", untar.toURI().toString(), hbaseRootDir.toString()});
127 doFsCommand(shell, new String [] {"-lsr", "/"});
128
129 Configuration toolConf = TEST_UTIL.getConfiguration();
130 conf.set(HConstants.HBASE_DIR, TEST_UTIL.getDefaultRootDirPath().toString());
131 ToolRunner.run(toolConf, new NamespaceUpgrade(), new String[]{"--upgrade"});
132 assertTrue(FSUtils.getVersion(fs, hbaseRootDir).equals(HConstants.FILE_SYSTEM_VERSION));
133 doFsCommand(shell, new String [] {"-lsr", "/"});
134 TEST_UTIL.startMiniHBaseCluster(1, 1);
135
136 for(TableName table: tables) {
137 int count = 0;
138 for(Result res: new HTable(TEST_UTIL.getConfiguration(), table).getScanner(new Scan())) {
139 assertEquals(currentKeys[count++], Bytes.toString(res.getRow()));
140 }
141 Assert.assertEquals(currentKeys.length, count);
142 }
143 assertEquals(2, TEST_UTIL.getHBaseAdmin().listNamespaceDescriptors().length);
144
145
146 HTable secureTable = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
147 ResultScanner scanner = secureTable.getScanner(new Scan());
148 int count = 0;
149 for(Result r : scanner) {
150 count++;
151 }
152 assertEquals(3, count);
153 assertFalse(TEST_UTIL.getHBaseAdmin().tableExists(TableName.valueOf("_acl_")));
154
155
156 List<HRegion> regions = TEST_UTIL.getMiniHBaseCluster().getRegions(secureTable.getName());
157 for(HRegion region : regions) {
158 assertEquals(1, region.getStores().size());
159 }
160 }
161
162 static File untar(final File testdir) throws IOException {
163
164 final String datafile = "TestNamespaceUpgrade";
165 File srcTarFile = new File(
166 System.getProperty("project.build.testSourceDirectory", "src/test") +
167 File.separator + "data" + File.separator + datafile + ".tgz");
168 File homedir = new File(testdir.toString());
169 File tgtUntarDir = new File(homedir, "hbase");
170 if (tgtUntarDir.exists()) {
171 if (!FileUtil.fullyDelete(tgtUntarDir)) {
172 throw new IOException("Failed delete of " + tgtUntarDir.toString());
173 }
174 }
175 if (!srcTarFile.exists()) {
176 throw new IOException(srcTarFile+" does not exist");
177 }
178 LOG.info("Untarring " + srcTarFile + " into " + homedir.toString());
179 FileUtil.unTar(srcTarFile, homedir);
180 Assert.assertTrue(tgtUntarDir.exists());
181 return tgtUntarDir;
182 }
183
184 private static void doFsCommand(final FsShell shell, final String [] args)
185 throws Exception {
186
187 int errcode = shell.run(args);
188 if (errcode != 0) throw new IOException("Failed put; errcode=" + errcode);
189 }
190
191 @AfterClass
192 public static void tearDownAfterClass() throws Exception {
193 TEST_UTIL.shutdownMiniCluster();
194 }
195
196 @Test (timeout=300000)
197 public void testSnapshots() throws IOException, InterruptedException {
198 String snapshots[][] = {snapshot1Keys, snapshot2Keys};
199 for(int i = 1; i <= snapshots.length; i++) {
200 for(TableName table: tables) {
201 TEST_UTIL.getHBaseAdmin().cloneSnapshot(table+"_snapshot"+i, TableName.valueOf(table+"_clone"+i));
202 FSUtils.logFileSystemState(FileSystem.get(TEST_UTIL.getConfiguration()),
203 FSUtils.getRootDir(TEST_UTIL.getConfiguration()),
204 LOG);
205 int count = 0;
206 for(Result res: new HTable(TEST_UTIL.getConfiguration(), table+"_clone"+i).getScanner(new
207 Scan())) {
208 assertEquals(snapshots[i-1][count++], Bytes.toString(res.getRow()));
209 }
210 Assert.assertEquals(table+"_snapshot"+i, snapshots[i-1].length, count);
211 }
212 }
213 }
214
215 @Test (timeout=300000)
216 public void testRenameUsingSnapshots() throws Exception {
217 String newNS = "newNS";
218 TEST_UTIL.getHBaseAdmin().createNamespace(NamespaceDescriptor.create(newNS).build());
219 for(TableName table: tables) {
220 int count = 0;
221 for(Result res: new HTable(TEST_UTIL.getConfiguration(), table).getScanner(new
222 Scan())) {
223 assertEquals(currentKeys[count++], Bytes.toString(res.getRow()));
224 }
225 TEST_UTIL.getHBaseAdmin().snapshot(table + "_snapshot3", table);
226 final TableName newTableName =
227 TableName.valueOf(newNS + TableName.NAMESPACE_DELIM + table + "_clone3");
228 TEST_UTIL.getHBaseAdmin().cloneSnapshot(table + "_snapshot3", newTableName);
229 Thread.sleep(1000);
230 count = 0;
231 for(Result res: new HTable(TEST_UTIL.getConfiguration(), newTableName).getScanner(new
232 Scan())) {
233 assertEquals(currentKeys[count++], Bytes.toString(res.getRow()));
234 }
235 FSUtils.logFileSystemState(TEST_UTIL.getTestFileSystem(), TEST_UTIL.getDefaultRootDirPath()
236 , LOG);
237 Assert.assertEquals(newTableName + "", currentKeys.length, count);
238 TEST_UTIL.getHBaseAdmin().flush(newTableName);
239 TEST_UTIL.getHBaseAdmin().majorCompact(newTableName);
240 TEST_UTIL.waitFor(30000, new Waiter.Predicate<IOException>() {
241 @Override
242 public boolean evaluate() throws IOException {
243 return TEST_UTIL.getHBaseAdmin().getCompactionState(newTableName) ==
244 AdminProtos.GetRegionInfoResponse.CompactionState.NONE;
245 }
246 });
247 }
248
249 String nextNS = "nextNS";
250 TEST_UTIL.getHBaseAdmin().createNamespace(NamespaceDescriptor.create(nextNS).build());
251 for(TableName table: tables) {
252 TableName srcTable = TableName.valueOf(newNS + TableName.NAMESPACE_DELIM + table + "_clone3");
253 TEST_UTIL.getHBaseAdmin().snapshot(table + "_snapshot4", srcTable);
254 TableName newTableName =
255 TableName.valueOf(nextNS + TableName.NAMESPACE_DELIM + table + "_clone4");
256 TEST_UTIL.getHBaseAdmin().cloneSnapshot(table+"_snapshot4", newTableName);
257 FSUtils.logFileSystemState(TEST_UTIL.getTestFileSystem(), TEST_UTIL.getDefaultRootDirPath(),
258 LOG);
259 int count = 0;
260 for(Result res: new HTable(TEST_UTIL.getConfiguration(), newTableName).getScanner(new
261 Scan())) {
262 assertEquals(currentKeys[count++], Bytes.toString(res.getRow()));
263 }
264 Assert.assertEquals(newTableName + "", currentKeys.length, count);
265 }
266 }
267
268 @Test (timeout=300000)
269 public void testOldDirsAreGonePostMigration() throws IOException {
270 FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
271 Path hbaseRootDir = TEST_UTIL.getDefaultRootDirPath();
272 List <String> dirs = new ArrayList<String>(NamespaceUpgrade.NON_USER_TABLE_DIRS);
273
274 dirs.remove(HConstants.HBCK_SIDELINEDIR_NAME);
275 dirs.remove(HConstants.SNAPSHOT_DIR_NAME);
276 dirs.remove(HConstants.HBASE_TEMP_DIRECTORY);
277 for (String dir: dirs) {
278 assertFalse(fs.exists(new Path(hbaseRootDir, dir)));
279 }
280 }
281
282 @Test (timeout=300000)
283 public void testNewDirsArePresentPostMigration() throws IOException {
284 FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
285
286 String [] newdirs = new String [] {HConstants.BASE_NAMESPACE_DIR,
287 HConstants.HREGION_LOGDIR_NAME};
288 Path hbaseRootDir = TEST_UTIL.getDefaultRootDirPath();
289 for (String dir: newdirs) {
290 assertTrue(dir, fs.exists(new Path(hbaseRootDir, dir)));
291 }
292 }
293
294 @Test (timeout = 300000)
295 public void testACLTableMigration() throws IOException {
296 Path rootDir = TEST_UTIL.getDataTestDirOnTestFS("testACLTable");
297 FileSystem fs = TEST_UTIL.getTestFileSystem();
298 Configuration conf = TEST_UTIL.getConfiguration();
299 byte[] FAMILY = Bytes.toBytes("l");
300 byte[] QUALIFIER = Bytes.toBytes("testUser");
301 byte[] VALUE = Bytes.toBytes("RWCA");
302
303
304 HTableDescriptor aclTable = new HTableDescriptor(TableName.valueOf("testACLTable"));
305 aclTable.addFamily(new HColumnDescriptor(FAMILY));
306 FSTableDescriptors fstd = new FSTableDescriptors(conf, fs, rootDir);
307 fstd.createTableDescriptor(aclTable);
308 HRegionInfo hriAcl = new HRegionInfo(aclTable.getTableName(), null, null);
309 HRegion region = HRegion.createHRegion(hriAcl, rootDir, conf, aclTable);
310 try {
311
312 Put p = new Put(Bytes.toBytes("-ROOT-"));
313 p.addImmutable(FAMILY, QUALIFIER, VALUE);
314 region.put(p);
315 p = new Put(Bytes.toBytes(".META."));
316 p.addImmutable(FAMILY, QUALIFIER, VALUE);
317 region.put(p);
318 p = new Put(Bytes.toBytes("_acl_"));
319 p.addImmutable(FAMILY, QUALIFIER, VALUE);
320 region.put(p);
321
322 NamespaceUpgrade upgrade = new NamespaceUpgrade();
323 upgrade.updateAcls(region);
324
325
326 Get g = new Get(Bytes.toBytes("-ROOT-"));
327 Result r = region.get(g);
328 assertTrue(r == null || r.size() == 0);
329
330
331 g = new Get(AccessControlLists.ACL_TABLE_NAME.toBytes());
332 r = region.get(g);
333 assertTrue(r != null && r.size() == 1);
334 assertTrue(Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)) == 0);
335
336
337 g = new Get(TableName.META_TABLE_NAME.toBytes());
338 r = region.get(g);
339 assertTrue(r != null && r.size() == 1);
340 assertTrue(Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)) == 0);
341 } finally {
342 region.close();
343
344 HRegionFileSystem.deleteRegionFromFileSystem(conf, fs,
345 FSUtils.getTableDir(rootDir, hriAcl.getTable()), hriAcl);
346 }
347 }
348 }