View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.filter;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertTrue;
25  
26  import java.io.IOException;
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.List;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.hadoop.hbase.Cell;
34  import org.apache.hadoop.hbase.HBaseTestingUtility;
35  import org.apache.hadoop.hbase.HColumnDescriptor;
36  import org.apache.hadoop.hbase.HRegionInfo;
37  import org.apache.hadoop.hbase.HTableDescriptor;
38  import org.apache.hadoop.hbase.KeyValue;
39  import org.apache.hadoop.hbase.TableName;
40  import org.apache.hadoop.hbase.client.Put;
41  import org.apache.hadoop.hbase.client.Scan;
42  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
43  import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
44  import org.apache.hadoop.hbase.regionserver.HRegion;
45  import org.apache.hadoop.hbase.regionserver.InternalScanner;
46  import org.apache.hadoop.hbase.testclassification.SmallTests;
47  import org.apache.hadoop.hbase.util.Bytes;
48  import org.junit.After;
49  import org.junit.Before;
50  import org.junit.Test;
51  import org.junit.experimental.categories.Category;
52  
53  @Category(SmallTests.class)
54  public class TestDependentColumnFilter {
55    private static final Log LOG = LogFactory.getLog(TestDependentColumnFilter.class);
56    private static final byte[][] ROWS = {
57      Bytes.toBytes("test1"),Bytes.toBytes("test2")
58    };
59    private static final byte[][] FAMILIES = {
60      Bytes.toBytes("familyOne"),Bytes.toBytes("familyTwo")
61    };
62    private static final long STAMP_BASE = System.currentTimeMillis();
63    private static final long[] STAMPS = {
64      STAMP_BASE-100, STAMP_BASE-200, STAMP_BASE-300
65    };
66    private static final byte[] QUALIFIER = Bytes.toBytes("qualifier");
67    private static final byte[][] BAD_VALS = {
68      Bytes.toBytes("bad1"), Bytes.toBytes("bad2"), Bytes.toBytes("bad3")
69    };
70    private static final byte[] MATCH_VAL = Bytes.toBytes("match");
71    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
72  
73    List<KeyValue> testVals;
74    private HRegion region;
75  
76    @Before
77    public void setUp() throws Exception {
78      testVals = makeTestVals();
79  
80      HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(this.getClass().getSimpleName()));
81      HColumnDescriptor hcd0 = new HColumnDescriptor(FAMILIES[0]);
82      hcd0.setMaxVersions(3);
83      htd.addFamily(hcd0);
84      HColumnDescriptor hcd1 = new HColumnDescriptor(FAMILIES[1]);
85      hcd1.setMaxVersions(3);
86      htd.addFamily(hcd1);
87      HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
88      this.region = HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(),
89        TEST_UTIL.getConfiguration(), htd);
90      addData();
91    }
92  
93    @After
94    public void tearDown() throws Exception {
95      HRegion.closeHRegion(this.region);
96    }
97  
98    private void addData() throws IOException {
99      Put put = new Put(ROWS[0]);
100     // add in an entry for each stamp, with 2 as a "good" value
101     put.add(FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]);
102     put.add(FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1]);
103     put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
104     // add in entries for stamps 0 and 2.
105     // without a value check both will be "accepted"
106     // with one 2 will be accepted(since the corresponding ts entry
107     // has a matching value
108     put.add(FAMILIES[1], QUALIFIER, STAMPS[0], BAD_VALS[0]);
109     put.add(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]);
110 
111     this.region.put(put);
112 
113     put = new Put(ROWS[1]);
114     put.add(FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]);
115     // there is no corresponding timestamp for this so it should never pass
116     put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
117     // if we reverse the qualifiers this one should pass
118     put.add(FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL);
119     // should pass
120     put.add(FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2]);
121 
122     this.region.put(put);
123   }
124 
125   private List<KeyValue> makeTestVals() {
126     List<KeyValue> testVals = new ArrayList<KeyValue>();
127     testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]));
128     testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1]));
129     testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2]));
130     testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL));
131     testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]));
132 
133     return testVals;
134   }
135 
136   /**
137    * This shouldn't be confused with TestFilter#verifyScan
138    * as expectedKeys is not the per row total, but the scan total
139    *
140    * @param s
141    * @param expectedRows
142    * @param expectedCells
143    * @throws IOException
144    */
145   private void verifyScan(Scan s, long expectedRows, long expectedCells)
146   throws IOException {
147     InternalScanner scanner = this.region.getScanner(s);
148     List<Cell> results = new ArrayList<Cell>();
149     int i = 0;
150     int cells = 0;
151     for (boolean done = true; done; i++) {
152       done = scanner.next(results);
153       Arrays.sort(results.toArray(new KeyValue[results.size()]),
154           KeyValue.COMPARATOR);
155       LOG.info("counter=" + i + ", " + results);
156       if (results.isEmpty()) break;
157       cells += results.size();
158       assertTrue("Scanned too many rows! Only expected " + expectedRows +
159           " total but already scanned " + (i+1), expectedRows > i);
160       assertTrue("Expected " + expectedCells + " cells total but " +
161           "already scanned " + cells, expectedCells >= cells);
162       results.clear();
163     }
164     assertEquals("Expected " + expectedRows + " rows but scanned " + i +
165         " rows", expectedRows, i);
166     assertEquals("Expected " + expectedCells + " cells but scanned " + cells +
167             " cells", expectedCells, cells);
168   }
169 
170   /**
171    * Test scans using a DependentColumnFilter
172    */
173   @Test
174   public void testScans() throws Exception {
175     Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
176 
177     Scan scan = new Scan();
178     scan.setFilter(filter);
179     scan.setMaxVersions(Integer.MAX_VALUE);
180 
181     verifyScan(scan, 2, 8);
182 
183     // drop the filtering cells
184     filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true);
185     scan = new Scan();
186     scan.setFilter(filter);
187     scan.setMaxVersions(Integer.MAX_VALUE);
188 
189     verifyScan(scan, 2, 3);
190 
191     // include a comparator operation
192     filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, false,
193         CompareOp.EQUAL, new BinaryComparator(MATCH_VAL));
194     scan = new Scan();
195     scan.setFilter(filter);
196     scan.setMaxVersions(Integer.MAX_VALUE);
197 
198     /*
199      * expecting to get the following 3 cells
200      * row 0
201      *   put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
202      *   put.add(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]);
203      * row 1
204      *   put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
205      */
206     verifyScan(scan, 2, 3);
207 
208     // include a comparator operation and drop comparator
209     filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true,
210         CompareOp.EQUAL, new BinaryComparator(MATCH_VAL));
211     scan = new Scan();
212     scan.setFilter(filter);
213     scan.setMaxVersions(Integer.MAX_VALUE);
214 
215     /*
216      * expecting to get the following 1 cell
217      * row 0
218      *   put.add(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]);
219      */
220     verifyScan(scan, 1, 1);
221 
222   }
223 
224   /**
225    * Test that the filter correctly drops rows without a corresponding timestamp
226    *
227    * @throws Exception
228    */
229   @Test
230   public void testFilterDropping() throws Exception {
231     Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
232     List<Cell> accepted = new ArrayList<Cell>();
233     for(Cell val : testVals) {
234       if(filter.filterKeyValue(val) == ReturnCode.INCLUDE) {
235         accepted.add(val);
236       }
237     }
238     assertEquals("check all values accepted from filterKeyValue", 5, accepted.size());
239 
240     filter.filterRowCells(accepted);
241     assertEquals("check filterRow(List<KeyValue>) dropped cell without corresponding column entry", 4, accepted.size());
242 
243     // start do it again with dependent column dropping on
244     filter = new DependentColumnFilter(FAMILIES[1], QUALIFIER, true);
245     accepted.clear();
246     for(KeyValue val : testVals) {
247         if(filter.filterKeyValue(val) == ReturnCode.INCLUDE) {
248           accepted.add(val);
249         }
250       }
251       assertEquals("check the filtering column cells got dropped", 2, accepted.size());
252 
253       filter.filterRowCells(accepted);
254       assertEquals("check cell retention", 2, accepted.size());
255   }
256 
257   /**
258    * Test for HBASE-8794. Avoid NullPointerException in DependentColumnFilter.toString().
259    */
260   @Test
261   public void testToStringWithNullComparator() {
262     // Test constructor that implicitly sets a null comparator
263     Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
264     assertNotNull(filter.toString());
265     assertTrue("check string contains 'null' as compatator is null",
266       filter.toString().contains("null"));
267 
268     // Test constructor with explicit null comparator
269     filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOp.EQUAL, null);
270     assertNotNull(filter.toString());
271     assertTrue("check string contains 'null' as compatator is null",
272       filter.toString().contains("null"));
273   }
274 
275   @Test
276   public void testToStringWithNonNullComparator() {
277     Filter filter =
278         new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOp.EQUAL,
279             new BinaryComparator(MATCH_VAL));
280     assertNotNull(filter.toString());
281     assertTrue("check string contains comparator value", filter.toString().contains("match"));
282   }
283 
284 }
285