View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.regionserver.compactions;
20  
21  import org.apache.commons.logging.Log;
22  import org.apache.commons.logging.LogFactory;
23  import org.apache.hadoop.conf.Configuration;
24  import org.apache.hadoop.hbase.HBaseConfiguration;
25  import org.apache.hadoop.hbase.testclassification.MediumTests;
26  import org.apache.hadoop.hbase.regionserver.HStore;
27  import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
28  import org.apache.hadoop.hbase.regionserver.StoreFile;
29  import org.apache.hadoop.hbase.util.ReflectionUtils;
30  import org.junit.Test;
31  import org.junit.experimental.categories.Category;
32  import org.junit.runner.RunWith;
33  import org.junit.runners.Parameterized;
34  
35  import java.io.IOException;
36  import java.util.ArrayList;
37  import java.util.Collection;
38  import java.util.List;
39  
40  import static org.mockito.Mockito.mock;
41  
42  import static org.mockito.Mockito.when;
43  
44  @Category(MediumTests.class)
45  @RunWith(Parameterized.class)
46  public class PerfTestCompactionPolicies extends MockStoreFileGenerator {
47  
48  
49    private static final Log LOG = LogFactory.getLog(PerfTestCompactionPolicies.class);
50  
51    private final RatioBasedCompactionPolicy cp;
52    private final StoreFileListGenerator generator;
53    private final HStore store;
54    private Class<? extends StoreFileListGenerator> fileGenClass;
55    private final int max;
56    private final int min;
57    private final float ratio;
58    private long written = 0;
59  
60    @Parameterized.Parameters
61    public static Collection<Object[]> data() {
62  
63  
64  
65      Class[] policyClasses = new Class[]{
66          EverythingPolicy.class,
67          RatioBasedCompactionPolicy.class,
68          ExploringCompactionPolicy.class,
69      };
70  
71      Class[] fileListGenClasses = new Class[]{
72          ExplicitFileListGenerator.class,
73          ConstantSizeFileListGenerator.class,
74          SemiConstantSizeFileListGenerator.class,
75          GaussianFileListGenerator.class,
76          SinusoidalFileListGenerator.class,
77          SpikyFileListGenerator.class
78       };
79  
80      int[] maxFileValues = new int[] {10};
81      int[] minFilesValues = new int[] {3};
82      float[] ratioValues = new float[] {1.2f};
83  
84      List<Object[]> params = new ArrayList<Object[]>(
85          maxFileValues.length
86          * minFilesValues.length
87          * fileListGenClasses.length
88          * policyClasses.length);
89  
90  
91      for (Class policyClass :  policyClasses) {
92        for (Class genClass: fileListGenClasses) {
93          for (int maxFile:maxFileValues) {
94            for (int minFile:minFilesValues) {
95              for (float ratio:ratioValues) {
96                params.add(new Object[] {policyClass, genClass, maxFile, minFile, ratio});
97              }
98            }
99          }
100       }
101     }
102 
103     return params;
104   }
105 
106   /**
107    * Test the perf of a CompactionPolicy with settings.
108    * @param cpClass The compaction policy to test
109    * @param inMmax The maximum number of file to compact
110    * @param inMin The min number of files to compact
111    * @param inRatio The ratio that files must be under to be compacted.
112    */
113   public PerfTestCompactionPolicies(
114       final Class<? extends CompactionPolicy> cpClass,
115       final Class<? extends StoreFileListGenerator> fileGenClass,
116       final int inMmax,
117       final int inMin,
118       final float inRatio) throws IllegalAccessException, InstantiationException {
119     super(PerfTestCompactionPolicies.class);
120     this.fileGenClass = fileGenClass;
121     this.max = inMmax;
122     this.min = inMin;
123     this.ratio = inRatio;
124 
125     // Hide lots of logging so the system out is usable as a tab delimited file.
126     org.apache.log4j.Logger.getLogger(CompactionConfiguration.class).
127         setLevel(org.apache.log4j.Level.ERROR);
128     org.apache.log4j.Logger.getLogger(RatioBasedCompactionPolicy.class).
129         setLevel(org.apache.log4j.Level.ERROR);
130 
131     org.apache.log4j.Logger.getLogger(cpClass).setLevel(org.apache.log4j.Level.ERROR);
132 
133 
134     Configuration configuration = HBaseConfiguration.create();
135 
136     // Make sure that this doesn't include every file.
137     configuration.setInt("hbase.hstore.compaction.max", max);
138     configuration.setInt("hbase.hstore.compaction.min", min);
139     configuration.setFloat("hbase.hstore.compaction.ratio", ratio);
140 
141     store = createMockStore();
142     this.cp = ReflectionUtils.instantiateWithCustomCtor(cpClass.getName(),
143         new Class[] {Configuration.class, StoreConfigInformation.class },
144         new Object[] {configuration, store });
145 
146     this.generator = fileGenClass.newInstance();
147     // Used for making paths
148   }
149 
150   @Test
151   public final void testSelection() throws Exception {
152     long fileDiff = 0;
153     for (List<StoreFile> storeFileList : generator) {
154       List<StoreFile> currentFiles = new ArrayList<StoreFile>(18);
155       for (StoreFile file : storeFileList) {
156         currentFiles.add(file);
157         currentFiles = runIteration(currentFiles);
158       }
159       fileDiff += (storeFileList.size() - currentFiles.size());
160     }
161 
162     // print out tab delimited so that it can be used in excel/gdocs.
163     System.out.println(
164             cp.getClass().getSimpleName()
165             + "\t" + fileGenClass.getSimpleName()
166             + "\t" + max
167             + "\t" + min
168             + "\t" + ratio
169             + "\t" + written
170             + "\t" + fileDiff
171     );
172   }
173 
174 
175   private List<StoreFile> runIteration(List<StoreFile> startingStoreFiles) throws IOException {
176 
177     List<StoreFile> storeFiles = new ArrayList<StoreFile>(startingStoreFiles);
178     CompactionRequest req = cp.selectCompaction(
179         storeFiles, new ArrayList<StoreFile>(), false, false, false);
180     long newFileSize = 0;
181 
182     Collection<StoreFile> filesToCompact = req.getFiles();
183 
184     if (!filesToCompact.isEmpty()) {
185 
186       storeFiles = new ArrayList<StoreFile>(storeFiles);
187       storeFiles.removeAll(filesToCompact);
188 
189       for (StoreFile storeFile : filesToCompact) {
190         newFileSize += storeFile.getReader().length();
191       }
192 
193       storeFiles.add(createMockStoreFileBytes(newFileSize));
194     }
195 
196     written += newFileSize;
197     return storeFiles;
198   }
199 
200   private HStore createMockStore() {
201     HStore s = mock(HStore.class);
202     when(s.getStoreFileTtl()).thenReturn(Long.MAX_VALUE);
203     when(s.getBlockingFileCount()).thenReturn(7L);
204     return s;
205   }
206 
207 }