View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.master.procedure;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertTrue;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.hbase.HBaseTestingUtility;
29  import org.apache.hadoop.hbase.HColumnDescriptor;
30  import org.apache.hadoop.hbase.HConstants;
31  import org.apache.hadoop.hbase.HRegionInfo;
32  import org.apache.hadoop.hbase.HTableDescriptor;
33  import org.apache.hadoop.hbase.TableName;
34  import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
35  import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
36  import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ModifyTableState;
37  import org.apache.hadoop.hbase.testclassification.MediumTests;
38  import org.junit.After;
39  import org.junit.AfterClass;
40  import org.junit.Before;
41  import org.junit.BeforeClass;
42  import org.junit.Test;
43  import org.junit.experimental.categories.Category;
44  
45  @Category(MediumTests.class)
46  public class TestModifyTableProcedure {
47    private static final Log LOG = LogFactory.getLog(TestModifyTableProcedure.class);
48  
49    protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
50  
51    private static long nonceGroup = HConstants.NO_NONCE;
52    private static long nonce = HConstants.NO_NONCE;
53  
54    private static void setupConf(Configuration conf) {
55      conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
56    }
57  
58    @BeforeClass
59    public static void setupCluster() throws Exception {
60      setupConf(UTIL.getConfiguration());
61      UTIL.startMiniCluster(1);
62    }
63  
64    @AfterClass
65    public static void cleanupTest() throws Exception {
66      try {
67        UTIL.shutdownMiniCluster();
68      } catch (Exception e) {
69        LOG.warn("failure shutting down cluster", e);
70      }
71    }
72  
73    @Before
74    public void setup() throws Exception {
75      ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
76      nonceGroup =
77          MasterProcedureTestingUtility.generateNonceGroup(UTIL.getHBaseCluster().getMaster());
78      nonce = MasterProcedureTestingUtility.generateNonce(UTIL.getHBaseCluster().getMaster());
79    }
80  
81    @After
82    public void tearDown() throws Exception {
83      ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
84      for (HTableDescriptor htd: UTIL.getHBaseAdmin().listTables()) {
85        LOG.info("Tear down, remove table=" + htd.getTableName());
86        UTIL.deleteTable(htd.getTableName());
87      }
88    }
89  
90    @Test(timeout=60000)
91    public void testModifyTable() throws Exception {
92      final TableName tableName = TableName.valueOf("testModifyTable");
93      final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
94  
95      MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf");
96      UTIL.getHBaseAdmin().disableTable(tableName);
97  
98      // Modify the table descriptor
99      HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
100 
101     // Test 1: Modify 1 property
102     long newMaxFileSize = htd.getMaxFileSize() * 2;
103     htd.setMaxFileSize(newMaxFileSize);
104     htd.setRegionReplication(3);
105 
106     long procId1 = ProcedureTestingUtility.submitAndWait(
107         procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
108     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1));
109 
110     HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
111     assertEquals(newMaxFileSize, currentHtd.getMaxFileSize());
112 
113     // Test 2: Modify multiple properties
114     boolean newReadOnlyOption = htd.isReadOnly() ? false : true;
115     long newMemStoreFlushSize = htd.getMemStoreFlushSize() * 2;
116     htd.setReadOnly(newReadOnlyOption);
117     htd.setMemStoreFlushSize(newMemStoreFlushSize);
118 
119     long procId2 = ProcedureTestingUtility.submitAndWait(
120         procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
121     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));
122 
123     currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
124     assertEquals(newReadOnlyOption, currentHtd.isReadOnly());
125     assertEquals(newMemStoreFlushSize, currentHtd.getMemStoreFlushSize());
126   }
127 
128   @Test(timeout = 60000)
129   public void testModifyTableAddCF() throws Exception {
130     final TableName tableName = TableName.valueOf("testModifyTableAddCF");
131     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
132 
133     MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf1");
134     HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
135     assertEquals(1, currentHtd.getFamiliesKeys().size());
136 
137     // Test 1: Modify the table descriptor online
138     String cf2 = "cf2";
139     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
140     htd.addFamily(new HColumnDescriptor(cf2));
141 
142     long procId = ProcedureTestingUtility.submitAndWait(
143         procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
144     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
145 
146     currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
147     assertEquals(2, currentHtd.getFamiliesKeys().size());
148     assertTrue(currentHtd.hasFamily(cf2.getBytes()));
149 
150     // Test 2: Modify the table descriptor offline
151     UTIL.getHBaseAdmin().disableTable(tableName);
152     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
153     String cf3 = "cf3";
154     HTableDescriptor htd2 =
155         new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
156     htd2.addFamily(new HColumnDescriptor(cf3));
157 
158     long procId2 =
159         ProcedureTestingUtility.submitAndWait(procExec,
160           new ModifyTableProcedure(procExec.getEnvironment(), htd2));
161     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));
162 
163     currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
164     assertTrue(currentHtd.hasFamily(cf3.getBytes()));
165     assertEquals(3, currentHtd.getFamiliesKeys().size());
166   }
167 
168   @Test(timeout = 60000)
169   public void testModifyTableDeleteCF() throws Exception {
170     final TableName tableName = TableName.valueOf("testModifyTableAddCF");
171     final String cf2 = "cf2";
172     final String cf3 = "cf3";
173     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
174 
175     MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf1", cf2, cf3);
176     HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
177     assertEquals(3, currentHtd.getFamiliesKeys().size());
178 
179     // Test 1: Modify the table descriptor
180     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
181     htd.removeFamily(cf2.getBytes());
182 
183     long procId = ProcedureTestingUtility.submitAndWait(
184         procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
185     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
186 
187     currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
188     assertEquals(2, currentHtd.getFamiliesKeys().size());
189     assertFalse(currentHtd.hasFamily(cf2.getBytes()));
190 
191     // Test 2: Modify the table descriptor offline
192     UTIL.getHBaseAdmin().disableTable(tableName);
193     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
194 
195     HTableDescriptor htd2 =
196         new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
197     htd2.removeFamily(cf3.getBytes());
198 
199     long procId2 =
200         ProcedureTestingUtility.submitAndWait(procExec,
201           new ModifyTableProcedure(procExec.getEnvironment(), htd2));
202     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));
203 
204     currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
205     assertEquals(1, currentHtd.getFamiliesKeys().size());
206     assertFalse(currentHtd.hasFamily(cf3.getBytes()));
207   }
208 
209   @Test(timeout=60000)
210   public void testRecoveryAndDoubleExecutionOffline() throws Exception {
211     final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecutionOffline");
212     final String cf2 = "cf2";
213     final String cf3 = "cf3";
214     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
215 
216     // create the table
217     HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
218       procExec, tableName, null, "cf1", cf3);
219     UTIL.getHBaseAdmin().disableTable(tableName);
220 
221     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
222     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
223 
224     // Modify multiple properties of the table.
225     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
226     boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
227     htd.setCompactionEnabled(newCompactionEnableOption);
228     htd.addFamily(new HColumnDescriptor(cf2));
229     htd.removeFamily(cf3.getBytes());
230     htd.setRegionReplication(3);
231 
232     // Start the Modify procedure && kill the executor
233     long procId = procExec.submitProcedure(
234       new ModifyTableProcedure(procExec.getEnvironment(), htd), nonceGroup, nonce);
235 
236     // Restart the executor and execute the step twice
237     int numberOfSteps = ModifyTableState.values().length;
238     MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(
239       procExec,
240       procId,
241       numberOfSteps,
242       ModifyTableState.values());
243 
244     // Validate descriptor
245     HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
246     assertEquals(newCompactionEnableOption, currentHtd.isCompactionEnabled());
247     assertEquals(2, currentHtd.getFamiliesKeys().size());
248 
249     // cf2 should be added cf3 should be removed
250     MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
251       tableName, regions, false, "cf1", cf2);
252   }
253 
254   @Test(timeout = 60000)
255   public void testRecoveryAndDoubleExecutionOnline() throws Exception {
256     final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecutionOnline");
257     final String cf2 = "cf2";
258     final String cf3 = "cf3";
259     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
260 
261     // create the table
262     HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
263       procExec, tableName, null, "cf1", cf3);
264 
265     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
266 
267     // Modify multiple properties of the table.
268     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
269     boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
270     htd.setCompactionEnabled(newCompactionEnableOption);
271     htd.addFamily(new HColumnDescriptor(cf2));
272     htd.removeFamily(cf3.getBytes());
273 
274     // Start the Modify procedure && kill the executor
275     long procId = procExec.submitProcedure(
276       new ModifyTableProcedure(procExec.getEnvironment(), htd), nonceGroup, nonce);
277 
278     // Restart the executor and execute the step twice
279     int numberOfSteps = ModifyTableState.values().length;
280     MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, numberOfSteps,
281       ModifyTableState.values());
282 
283     // Validate descriptor
284     HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
285     assertEquals(newCompactionEnableOption, currentHtd.isCompactionEnabled());
286     assertEquals(2, currentHtd.getFamiliesKeys().size());
287     assertTrue(currentHtd.hasFamily(cf2.getBytes()));
288     assertFalse(currentHtd.hasFamily(cf3.getBytes()));
289 
290     // cf2 should be added cf3 should be removed
291     MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
292       tableName, regions, "cf1", cf2);
293   }
294 
295   @Test(timeout = 60000)
296   public void testRollbackAndDoubleExecutionOnline() throws Exception {
297     final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution");
298     final String familyName = "cf2";
299     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
300 
301     // create the table
302     HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
303       procExec, tableName, null, "cf1");
304 
305     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
306 
307     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
308     boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
309     htd.setCompactionEnabled(newCompactionEnableOption);
310     htd.addFamily(new HColumnDescriptor(familyName));
311 
312     // Start the Modify procedure && kill the executor
313     long procId = procExec.submitProcedure(
314       new ModifyTableProcedure(procExec.getEnvironment(), htd), nonceGroup, nonce);
315 
316     // Restart the executor and rollback the step twice
317     int numberOfSteps = ModifyTableState.values().length - 4; // failing in the middle of proc
318     MasterProcedureTestingUtility.testRollbackAndDoubleExecution(
319       procExec,
320       procId,
321       numberOfSteps,
322       ModifyTableState.values());
323 
324     // cf2 should not be present
325     MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
326       tableName, regions, "cf1");
327   }
328 
329   @Test(timeout = 60000)
330   public void testRollbackAndDoubleExecutionOffline() throws Exception {
331     final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution");
332     final String familyName = "cf2";
333     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
334 
335     // create the table
336     HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
337       procExec, tableName, null, "cf1");
338     UTIL.getHBaseAdmin().disableTable(tableName);
339 
340     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
341     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
342 
343     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
344     boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
345     htd.setCompactionEnabled(newCompactionEnableOption);
346     htd.addFamily(new HColumnDescriptor(familyName));
347     htd.setRegionReplication(3);
348 
349     // Start the Modify procedure && kill the executor
350     long procId = procExec.submitProcedure(
351       new ModifyTableProcedure(procExec.getEnvironment(), htd), nonceGroup, nonce);
352 
353     // Restart the executor and rollback the step twice
354     int numberOfSteps = ModifyTableState.values().length - 4; // failing in the middle of proc
355     MasterProcedureTestingUtility.testRollbackAndDoubleExecution(
356       procExec,
357       procId,
358       numberOfSteps,
359       ModifyTableState.values());
360 
361     // cf2 should not be present
362     MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
363       tableName, regions, "cf1");
364   }
365 
366   @Test(timeout = 60000)
367   public void testRollbackAndDoubleExecutionAfterPONR() throws Exception {
368     final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecutionAfterPONR");
369     final String familyToAddName = "cf2";
370     final String familyToRemove = "cf1";
371     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
372 
373     // create the table
374     HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
375       procExec, tableName, null, familyToRemove);
376     UTIL.getHBaseAdmin().disableTable(tableName);
377 
378     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
379     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
380 
381     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
382     htd.setCompactionEnabled(!htd.isCompactionEnabled());
383     htd.addFamily(new HColumnDescriptor(familyToAddName));
384     htd.removeFamily(familyToRemove.getBytes());
385     htd.setRegionReplication(3);
386 
387     // Start the Modify procedure && kill the executor
388     long procId = procExec.submitProcedure(
389       new ModifyTableProcedure(procExec.getEnvironment(), htd), nonceGroup, nonce);
390 
391     // Failing after MODIFY_TABLE_DELETE_FS_LAYOUT we should not trigger the rollback.
392     // NOTE: the 5 (number of MODIFY_TABLE_DELETE_FS_LAYOUT + 1 step) is hardcoded,
393     //       so you have to look at this test at least once when you add a new step.
394     int numberOfSteps = 5;
395     MasterProcedureTestingUtility.testRollbackAndDoubleExecutionAfterPONR(
396       procExec,
397       procId,
398       numberOfSteps,
399       ModifyTableState.values());
400 
401     // "cf2" should be added and "cf1" should be removed
402     MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
403       tableName, regions, false, familyToAddName);
404   }
405 
406   private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
407     return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
408   }
409 }