View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.client;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.fail;
23  
24  import java.io.IOException;
25  import java.util.HashSet;
26  import java.util.Set;
27  
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.CategoryBasedTimeout;
30  import org.apache.hadoop.hbase.HBaseTestingUtility;
31  import org.apache.hadoop.hbase.HColumnDescriptor;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.HTableDescriptor;
34  import org.apache.hadoop.hbase.testclassification.LargeTests;
35  import org.apache.hadoop.hbase.TableName;
36  import org.apache.hadoop.hbase.master.MasterFileSystem;
37  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
38  import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
39  import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
40  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
41  import org.apache.hadoop.hbase.util.Bytes;
42  import org.apache.hadoop.hbase.util.FSUtils;
43  import org.junit.After;
44  import org.junit.AfterClass;
45  import org.junit.Before;
46  import org.junit.BeforeClass;
47  import org.junit.Rule;
48  import org.junit.Test;
49  import org.junit.experimental.categories.Category;
50  import org.junit.rules.TestRule;
51  
52  /**
53   * Test restore snapshots from the client
54   */
55  @Category(LargeTests.class)
56  public class TestRestoreSnapshotFromClient {
57    @Rule public final TestRule timeout = CategoryBasedTimeout.builder()
58        .withTimeout(this.getClass())
59        .withLookingForStuckThread(true)
60        .build();
61  
62    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
63  
64    private final byte[] FAMILY = Bytes.toBytes("cf");
65  
66    private byte[] emptySnapshot;
67    private byte[] snapshotName0;
68    private byte[] snapshotName1;
69    private byte[] snapshotName2;
70    private int snapshot0Rows;
71    private int snapshot1Rows;
72    private TableName tableName;
73    private Admin admin;
74  
75    @BeforeClass
76    public static void setUpBeforeClass() throws Exception {
77      TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
78      TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
79      TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
80      TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
81      TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
82      TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
83      TEST_UTIL.getConfiguration().setBoolean(
84          "hbase.master.enabletable.roundrobin", true);
85      TEST_UTIL.startMiniCluster(3);
86    }
87  
88    @AfterClass
89    public static void tearDownAfterClass() throws Exception {
90      TEST_UTIL.shutdownMiniCluster();
91    }
92  
93    /**
94     * Initialize the tests with a table filled with some data
95     * and two snapshots (snapshotName0, snapshotName1) of different states.
96     * The tableName, snapshotNames and the number of rows in the snapshot are initialized.
97     */
98    @Before
99    public void setup() throws Exception {
100     this.admin = TEST_UTIL.getHBaseAdmin();
101 
102     long tid = System.currentTimeMillis();
103     tableName =
104         TableName.valueOf("testtb-" + tid);
105     emptySnapshot = Bytes.toBytes("emptySnaptb-" + tid);
106     snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
107     snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
108     snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
109 
110     // create Table and disable it
111     SnapshotTestingUtils.createTable(TEST_UTIL, tableName, getNumReplicas(), FAMILY);
112     admin.disableTable(tableName);
113 
114     // take an empty snapshot
115     admin.snapshot(emptySnapshot, tableName);
116 
117     // enable table and insert data
118     admin.enableTable(tableName);
119     SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
120     try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
121       snapshot0Rows = TEST_UTIL.countRows(table);
122     }
123     admin.disableTable(tableName);
124 
125     // take a snapshot
126     admin.snapshot(snapshotName0, tableName);
127 
128     // enable table and insert more data
129     admin.enableTable(tableName);
130     SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
131     try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
132       snapshot1Rows = TEST_UTIL.countRows(table);
133     }
134   }
135 
136   @After
137   public void tearDown() throws Exception {
138     TEST_UTIL.deleteTable(tableName);
139     SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin());
140     SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
141   }
142 
143   @Test
144   public void testRestoreSnapshot() throws IOException {
145     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
146     admin.disableTable(tableName);
147     admin.snapshot(snapshotName1, tableName);
148     // Restore from snapshot-0
149     admin.restoreSnapshot(snapshotName0);
150     admin.enableTable(tableName);
151     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
152     SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
153 
154     // Restore from emptySnapshot
155     admin.disableTable(tableName);
156     admin.restoreSnapshot(emptySnapshot);
157     admin.enableTable(tableName);
158     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, 0);
159     SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
160 
161     // Restore from snapshot-1
162     admin.disableTable(tableName);
163     admin.restoreSnapshot(snapshotName1);
164     admin.enableTable(tableName);
165     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
166     SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
167 
168     // Restore from snapshot-1
169     TEST_UTIL.deleteTable(tableName);
170     admin.restoreSnapshot(snapshotName1);
171     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
172     SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
173   }
174 
175   protected int getNumReplicas() {
176     return 1;
177   }
178 
179   @Test
180   public void testRestoreSchemaChange() throws Exception {
181     byte[] TEST_FAMILY2 = Bytes.toBytes("cf2");
182 
183     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
184 
185     // Add one column family and put some data in it
186     admin.disableTable(tableName);
187     admin.addColumn(tableName, new HColumnDescriptor(TEST_FAMILY2));
188     admin.enableTable(tableName);
189     assertEquals(2, table.getTableDescriptor().getFamilies().size());
190     HTableDescriptor htd = admin.getTableDescriptor(tableName);
191     assertEquals(2, htd.getFamilies().size());
192     SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, TEST_FAMILY2);
193     long snapshot2Rows = snapshot1Rows + 500;
194     assertEquals(snapshot2Rows, TEST_UTIL.countRows(table));
195     assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2));
196     Set<String> fsFamilies = getFamiliesFromFS(tableName);
197     assertEquals(2, fsFamilies.size());
198 
199     // Take a snapshot
200     admin.disableTable(tableName);
201     admin.snapshot(snapshotName2, tableName);
202 
203     // Restore the snapshot (without the cf)
204     admin.restoreSnapshot(snapshotName0);
205     admin.enableTable(tableName);
206     assertEquals(1, table.getTableDescriptor().getFamilies().size());
207     try {
208       TEST_UTIL.countRows(table, TEST_FAMILY2);
209       fail("family '" + Bytes.toString(TEST_FAMILY2) + "' should not exists");
210     } catch (NoSuchColumnFamilyException e) {
211       // expected
212     }
213     assertEquals(snapshot0Rows, TEST_UTIL.countRows(table));
214     htd = admin.getTableDescriptor(tableName);
215     assertEquals(1, htd.getFamilies().size());
216     fsFamilies = getFamiliesFromFS(tableName);
217     assertEquals(1, fsFamilies.size());
218 
219     // Restore back the snapshot (with the cf)
220     admin.disableTable(tableName);
221     admin.restoreSnapshot(snapshotName2);
222     admin.enableTable(tableName);
223     htd = admin.getTableDescriptor(tableName);
224     assertEquals(2, htd.getFamilies().size());
225     assertEquals(2, table.getTableDescriptor().getFamilies().size());
226     assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2));
227     assertEquals(snapshot2Rows, TEST_UTIL.countRows(table));
228     fsFamilies = getFamiliesFromFS(tableName);
229     assertEquals(2, fsFamilies.size());
230     table.close();
231   }
232 
233   @Test
234   public void testCloneSnapshotOfCloned() throws IOException, InterruptedException {
235     TableName clonedTableName =
236         TableName.valueOf("clonedtb-" + System.currentTimeMillis());
237     admin.cloneSnapshot(snapshotName0, clonedTableName);
238     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
239     SnapshotTestingUtils.verifyReplicasCameOnline(clonedTableName, admin, getNumReplicas());
240     admin.disableTable(clonedTableName);
241     admin.snapshot(snapshotName2, clonedTableName);
242     TEST_UTIL.deleteTable(clonedTableName);
243     waitCleanerRun();
244 
245     admin.cloneSnapshot(snapshotName2, clonedTableName);
246     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
247     SnapshotTestingUtils.verifyReplicasCameOnline(clonedTableName, admin, getNumReplicas());
248     TEST_UTIL.deleteTable(clonedTableName);
249   }
250 
251   @Test
252   public void testCloneAndRestoreSnapshot() throws IOException, InterruptedException {
253     TEST_UTIL.deleteTable(tableName);
254     waitCleanerRun();
255 
256     admin.cloneSnapshot(snapshotName0, tableName);
257     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
258     SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
259     waitCleanerRun();
260 
261     admin.disableTable(tableName);
262     admin.restoreSnapshot(snapshotName0);
263     admin.enableTable(tableName);
264     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
265     SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
266   }
267 
268   @Test
269   public void testCorruptedSnapshot() throws IOException, InterruptedException {
270     SnapshotTestingUtils.corruptSnapshot(TEST_UTIL, Bytes.toString(snapshotName0));
271     TableName cloneName = TableName.valueOf("corruptedClone-" + System.currentTimeMillis());
272     try {
273       admin.cloneSnapshot(snapshotName0, cloneName);
274       fail("Expected CorruptedSnapshotException, got succeeded cloneSnapshot()");
275     } catch (CorruptedSnapshotException e) {
276       // Got the expected corruption exception.
277       // check for no references of the cloned table.
278       assertFalse(admin.tableExists(cloneName));
279     } catch (Exception e) {
280       fail("Expected CorruptedSnapshotException got: " + e);
281     }
282   }
283 
284   // ==========================================================================
285   //  Helpers
286   // ==========================================================================
287   private void waitCleanerRun() throws InterruptedException {
288     TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
289   }
290 
291   private Set<String> getFamiliesFromFS(final TableName tableName) throws IOException {
292     MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
293     Set<String> families = new HashSet<String>();
294     Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName);
295     for (Path regionDir: FSUtils.getRegionDirs(mfs.getFileSystem(), tableDir)) {
296       for (Path familyDir: FSUtils.getFamilyDirs(mfs.getFileSystem(), regionDir)) {
297         families.add(familyDir.getName());
298       }
299     }
300     return families;
301   }
302 }