View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver.compactions;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  
23  import java.io.IOException;
24  import java.util.Collection;
25  import java.util.List;
26  import java.util.Random;
27  
28  import org.apache.hadoop.conf.Configuration;
29  import org.apache.hadoop.hbase.HBaseTestingUtility;
30  import org.apache.hadoop.hbase.HColumnDescriptor;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.HTableDescriptor;
33  import org.apache.hadoop.hbase.MiniHBaseCluster;
34  import org.apache.hadoop.hbase.TableName;
35  import org.apache.hadoop.hbase.client.HBaseAdmin;
36  import org.apache.hadoop.hbase.client.Put;
37  import org.apache.hadoop.hbase.client.Table;
38  import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
39  import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
40  import org.apache.hadoop.hbase.regionserver.HRegionServer;
41  import org.apache.hadoop.hbase.regionserver.HStore;
42  import org.apache.hadoop.hbase.regionserver.Region;
43  import org.apache.hadoop.hbase.regionserver.Store;
44  import org.apache.hadoop.hbase.regionserver.StoreFile;
45  import org.apache.hadoop.hbase.testclassification.MediumTests;
46  import org.apache.hadoop.hbase.util.Bytes;
47  import org.apache.hadoop.hbase.util.EnvironmentEdge;
48  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
49  import org.apache.hadoop.hbase.util.JVMClusterUtil;
50  import org.apache.hadoop.hbase.util.TimeOffsetEnvironmentEdge;
51  import org.junit.AfterClass;
52  import org.junit.Assert;
53  import org.junit.BeforeClass;
54  import org.junit.Test;
55  import org.junit.experimental.categories.Category;
56  
57  @Category({ MediumTests.class })
58  public class TestFIFOCompactionPolicy {
59  
60    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
61  
62  
63    private final TableName tableName = TableName.valueOf(getClass().getSimpleName());
64  
65    private final byte[] family = Bytes.toBytes("f");
66  
67    private final byte[] qualifier = Bytes.toBytes("q");
68  
69    private Store getStoreWithName(TableName tableName) {
70      MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster();
71      List<JVMClusterUtil.RegionServerThread> rsts = cluster.getRegionServerThreads();
72      for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) {
73        HRegionServer hrs = rsts.get(i).getRegionServer();
74        for (Region region : hrs.getOnlineRegions(tableName)) {
75          return region.getStores().iterator().next();
76        }
77      }
78      return null;
79    }
80  
81    private Store prepareData() throws IOException {
82      HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
83      if (admin.tableExists(tableName)) {
84        admin.disableTable(tableName);
85        admin.deleteTable(tableName);
86      }
87      HTableDescriptor desc = new HTableDescriptor(tableName);
88      desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, 
89        FIFOCompactionPolicy.class.getName());
90      desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, 
91        DisabledRegionSplitPolicy.class.getName());
92      HColumnDescriptor colDesc = new HColumnDescriptor(family);
93      colDesc.setTimeToLive(1); // 1 sec
94      desc.addFamily(colDesc);
95  
96      admin.createTable(desc);
97      Table table = TEST_UTIL.getConnection().getTable(tableName);
98      Random rand = new Random();
99      TimeOffsetEnvironmentEdge edge =
100         (TimeOffsetEnvironmentEdge) EnvironmentEdgeManager.getDelegate();
101     for (int i = 0; i < 10; i++) {
102       for (int j = 0; j < 10; j++) {
103         byte[] value = new byte[128 * 1024];
104         rand.nextBytes(value);
105         table.put(new Put(Bytes.toBytes(i * 10 + j)).addColumn(family, qualifier, value));
106       }
107       admin.flush(tableName);
108       edge.increment(1001);
109     }
110     return getStoreWithName(tableName);
111   }
112 
113   @BeforeClass   
114   public static void setEnvironmentEdge()
115   {
116     EnvironmentEdge ee = new TimeOffsetEnvironmentEdge();
117     EnvironmentEdgeManager.injectEdge(ee);
118   }
119   
120   @AfterClass
121   public static void resetEnvironmentEdge()
122   {
123     EnvironmentEdgeManager.reset();
124   }
125   
126   @Test
127   public void testPurgeExpiredFiles() throws Exception {
128     Configuration conf = TEST_UTIL.getConfiguration();
129     conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000);
130 
131     TEST_UTIL.startMiniCluster(1);
132     try {
133       Store store = prepareData();
134       assertEquals(10, store.getStorefilesCount());
135       TEST_UTIL.getHBaseAdmin().majorCompact(tableName);
136       while (store.getStorefilesCount() > 1) {
137         Thread.sleep(100);
138       }
139       assertTrue(store.getStorefilesCount() == 1);
140     } finally {
141       TEST_UTIL.shutdownMiniCluster();
142     }
143   }
144   
145   @Test  
146   public void testSanityCheckTTL() throws Exception
147   {
148     Configuration conf = TEST_UTIL.getConfiguration();
149     conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000);
150     TEST_UTIL.startMiniCluster(1);
151 
152     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
153     String tableName = this.tableName.getNameAsString()+"-TTL";
154     if (admin.tableExists(tableName)) {
155       admin.disableTable(tableName);
156       admin.deleteTable(tableName);
157     }
158     HTableDescriptor desc = new HTableDescriptor(tableName);
159     desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, 
160       FIFOCompactionPolicy.class.getName());
161     desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, 
162       DisabledRegionSplitPolicy.class.getName());
163     HColumnDescriptor colDesc = new HColumnDescriptor(family);
164     desc.addFamily(colDesc);
165     try{
166       admin.createTable(desc);
167       Assert.fail();
168     }catch(Exception e){      
169     }finally{
170       TEST_UTIL.shutdownMiniCluster();
171     }
172   }
173 
174   @Test  
175   public void testSanityCheckMinVersion() throws Exception
176   {
177     Configuration conf = TEST_UTIL.getConfiguration();
178     conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000);
179     TEST_UTIL.startMiniCluster(1);
180 
181     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
182     String tableName = this.tableName.getNameAsString()+"-MinVersion";
183     if (admin.tableExists(tableName)) {
184       admin.disableTable(tableName);
185       admin.deleteTable(tableName);
186     }
187     HTableDescriptor desc = new HTableDescriptor(tableName);
188     desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, 
189       FIFOCompactionPolicy.class.getName());
190     desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, 
191       DisabledRegionSplitPolicy.class.getName());
192     HColumnDescriptor colDesc = new HColumnDescriptor(family);
193     colDesc.setTimeToLive(1); // 1 sec
194     colDesc.setMinVersions(1);
195     desc.addFamily(colDesc);
196     try{
197       admin.createTable(desc);
198       Assert.fail();
199     }catch(Exception e){      
200     }finally{
201       TEST_UTIL.shutdownMiniCluster();
202     }
203   }
204   
205   @Test  
206   public void testSanityCheckBlockingStoreFiles() throws Exception
207   {
208     Configuration conf = TEST_UTIL.getConfiguration();
209     conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10);
210     TEST_UTIL.startMiniCluster(1);
211 
212     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
213     String tableName = this.tableName.getNameAsString()+"-MinVersion";
214     if (admin.tableExists(tableName)) {
215       admin.disableTable(tableName);
216       admin.deleteTable(tableName);
217     }
218     HTableDescriptor desc = new HTableDescriptor(tableName);
219     desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, 
220       FIFOCompactionPolicy.class.getName());
221     desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, 
222       DisabledRegionSplitPolicy.class.getName());
223     HColumnDescriptor colDesc = new HColumnDescriptor(family);
224     colDesc.setTimeToLive(1); // 1 sec
225     desc.addFamily(colDesc);
226     try{
227       admin.createTable(desc);
228       Assert.fail();
229     }catch(Exception e){      
230     }finally{
231       TEST_UTIL.shutdownMiniCluster();
232     }
233   }
234 }