1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver.compactions;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22
23 import java.io.IOException;
24 import java.util.Collection;
25 import java.util.List;
26 import java.util.Random;
27
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.hbase.HBaseTestingUtility;
30 import org.apache.hadoop.hbase.HColumnDescriptor;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.HTableDescriptor;
33 import org.apache.hadoop.hbase.MiniHBaseCluster;
34 import org.apache.hadoop.hbase.TableName;
35 import org.apache.hadoop.hbase.client.HBaseAdmin;
36 import org.apache.hadoop.hbase.client.Put;
37 import org.apache.hadoop.hbase.client.Table;
38 import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
39 import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
40 import org.apache.hadoop.hbase.regionserver.HRegionServer;
41 import org.apache.hadoop.hbase.regionserver.HStore;
42 import org.apache.hadoop.hbase.regionserver.Region;
43 import org.apache.hadoop.hbase.regionserver.Store;
44 import org.apache.hadoop.hbase.regionserver.StoreFile;
45 import org.apache.hadoop.hbase.testclassification.MediumTests;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.hbase.util.EnvironmentEdge;
48 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
49 import org.apache.hadoop.hbase.util.JVMClusterUtil;
50 import org.apache.hadoop.hbase.util.TimeOffsetEnvironmentEdge;
51 import org.junit.AfterClass;
52 import org.junit.Assert;
53 import org.junit.BeforeClass;
54 import org.junit.Test;
55 import org.junit.experimental.categories.Category;
56
57 @Category({ MediumTests.class })
58 public class TestFIFOCompactionPolicy {
59
60 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
61
62
63 private final TableName tableName = TableName.valueOf(getClass().getSimpleName());
64
65 private final byte[] family = Bytes.toBytes("f");
66
67 private final byte[] qualifier = Bytes.toBytes("q");
68
69 private Store getStoreWithName(TableName tableName) {
70 MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster();
71 List<JVMClusterUtil.RegionServerThread> rsts = cluster.getRegionServerThreads();
72 for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) {
73 HRegionServer hrs = rsts.get(i).getRegionServer();
74 for (Region region : hrs.getOnlineRegions(tableName)) {
75 return region.getStores().iterator().next();
76 }
77 }
78 return null;
79 }
80
81 private Store prepareData() throws IOException {
82 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
83 if (admin.tableExists(tableName)) {
84 admin.disableTable(tableName);
85 admin.deleteTable(tableName);
86 }
87 HTableDescriptor desc = new HTableDescriptor(tableName);
88 desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
89 FIFOCompactionPolicy.class.getName());
90 desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
91 DisabledRegionSplitPolicy.class.getName());
92 HColumnDescriptor colDesc = new HColumnDescriptor(family);
93 colDesc.setTimeToLive(1);
94 desc.addFamily(colDesc);
95
96 admin.createTable(desc);
97 Table table = TEST_UTIL.getConnection().getTable(tableName);
98 Random rand = new Random();
99 TimeOffsetEnvironmentEdge edge =
100 (TimeOffsetEnvironmentEdge) EnvironmentEdgeManager.getDelegate();
101 for (int i = 0; i < 10; i++) {
102 for (int j = 0; j < 10; j++) {
103 byte[] value = new byte[128 * 1024];
104 rand.nextBytes(value);
105 table.put(new Put(Bytes.toBytes(i * 10 + j)).addColumn(family, qualifier, value));
106 }
107 admin.flush(tableName);
108 edge.increment(1001);
109 }
110 return getStoreWithName(tableName);
111 }
112
113 @BeforeClass
114 public static void setEnvironmentEdge()
115 {
116 EnvironmentEdge ee = new TimeOffsetEnvironmentEdge();
117 EnvironmentEdgeManager.injectEdge(ee);
118 }
119
120 @AfterClass
121 public static void resetEnvironmentEdge()
122 {
123 EnvironmentEdgeManager.reset();
124 }
125
126 @Test
127 public void testPurgeExpiredFiles() throws Exception {
128 Configuration conf = TEST_UTIL.getConfiguration();
129 conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000);
130
131 TEST_UTIL.startMiniCluster(1);
132 try {
133 Store store = prepareData();
134 assertEquals(10, store.getStorefilesCount());
135 TEST_UTIL.getHBaseAdmin().majorCompact(tableName);
136 while (store.getStorefilesCount() > 1) {
137 Thread.sleep(100);
138 }
139 assertTrue(store.getStorefilesCount() == 1);
140 } finally {
141 TEST_UTIL.shutdownMiniCluster();
142 }
143 }
144
145 @Test
146 public void testSanityCheckTTL() throws Exception
147 {
148 Configuration conf = TEST_UTIL.getConfiguration();
149 conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000);
150 TEST_UTIL.startMiniCluster(1);
151
152 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
153 String tableName = this.tableName.getNameAsString()+"-TTL";
154 if (admin.tableExists(tableName)) {
155 admin.disableTable(tableName);
156 admin.deleteTable(tableName);
157 }
158 HTableDescriptor desc = new HTableDescriptor(tableName);
159 desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
160 FIFOCompactionPolicy.class.getName());
161 desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
162 DisabledRegionSplitPolicy.class.getName());
163 HColumnDescriptor colDesc = new HColumnDescriptor(family);
164 desc.addFamily(colDesc);
165 try{
166 admin.createTable(desc);
167 Assert.fail();
168 }catch(Exception e){
169 }finally{
170 TEST_UTIL.shutdownMiniCluster();
171 }
172 }
173
174 @Test
175 public void testSanityCheckMinVersion() throws Exception
176 {
177 Configuration conf = TEST_UTIL.getConfiguration();
178 conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000);
179 TEST_UTIL.startMiniCluster(1);
180
181 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
182 String tableName = this.tableName.getNameAsString()+"-MinVersion";
183 if (admin.tableExists(tableName)) {
184 admin.disableTable(tableName);
185 admin.deleteTable(tableName);
186 }
187 HTableDescriptor desc = new HTableDescriptor(tableName);
188 desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
189 FIFOCompactionPolicy.class.getName());
190 desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
191 DisabledRegionSplitPolicy.class.getName());
192 HColumnDescriptor colDesc = new HColumnDescriptor(family);
193 colDesc.setTimeToLive(1);
194 colDesc.setMinVersions(1);
195 desc.addFamily(colDesc);
196 try{
197 admin.createTable(desc);
198 Assert.fail();
199 }catch(Exception e){
200 }finally{
201 TEST_UTIL.shutdownMiniCluster();
202 }
203 }
204
205 @Test
206 public void testSanityCheckBlockingStoreFiles() throws Exception
207 {
208 Configuration conf = TEST_UTIL.getConfiguration();
209 conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10);
210 TEST_UTIL.startMiniCluster(1);
211
212 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
213 String tableName = this.tableName.getNameAsString()+"-MinVersion";
214 if (admin.tableExists(tableName)) {
215 admin.disableTable(tableName);
216 admin.deleteTable(tableName);
217 }
218 HTableDescriptor desc = new HTableDescriptor(tableName);
219 desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
220 FIFOCompactionPolicy.class.getName());
221 desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
222 DisabledRegionSplitPolicy.class.getName());
223 HColumnDescriptor colDesc = new HColumnDescriptor(family);
224 colDesc.setTimeToLive(1);
225 desc.addFamily(colDesc);
226 try{
227 admin.createTable(desc);
228 Assert.fail();
229 }catch(Exception e){
230 }finally{
231 TEST_UTIL.shutdownMiniCluster();
232 }
233 }
234 }