View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import static org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode.INCLUDE;
23  import static org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode.SKIP;
24  
25  import java.io.IOException;
26  import java.util.ArrayList;
27  import java.util.List;
28  import java.util.NavigableSet;
29  
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.HBaseConfiguration;
32  import org.apache.hadoop.hbase.HBaseTestCase;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.KeepDeletedCells;
35  import org.apache.hadoop.hbase.KeyValue;
36  import org.apache.hadoop.hbase.KeyValue.KVComparator;
37  import org.apache.hadoop.hbase.KeyValue.Type;
38  import org.apache.hadoop.hbase.testclassification.SmallTests;
39  import org.apache.hadoop.hbase.client.Get;
40  import org.apache.hadoop.hbase.client.Scan;
41  import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
44  import org.junit.experimental.categories.Category;
45  
46  @Category(SmallTests.class)
47  public class TestQueryMatcher extends HBaseTestCase {
48    private static final boolean PRINT = false;
49  
50    private byte[] row1;
51    private byte[] row2;
52    private byte[] row3;
53    private byte[] fam1;
54    private byte[] fam2;
55    private byte[] col1;
56    private byte[] col2;
57    private byte[] col3;
58    private byte[] col4;
59    private byte[] col5;
60  
61    private byte[] data;
62  
63    private Get get;
64  
65    long ttl = Long.MAX_VALUE;
66    KVComparator rowComparator;
67    private Scan scan;
68  
69    public void setUp() throws Exception {
70      super.setUp();
71      row1 = Bytes.toBytes("row1");
72      row2 = Bytes.toBytes("row2");
73      row3 = Bytes.toBytes("row3");
74      fam1 = Bytes.toBytes("fam1");
75      fam2 = Bytes.toBytes("fam2");
76      col1 = Bytes.toBytes("col1");
77      col2 = Bytes.toBytes("col2");
78      col3 = Bytes.toBytes("col3");
79      col4 = Bytes.toBytes("col4");
80      col5 = Bytes.toBytes("col5");
81  
82      data = Bytes.toBytes("data");
83  
84      //Create Get
85      get = new Get(row1);
86      get.addFamily(fam1);
87      get.addColumn(fam2, col2);
88      get.addColumn(fam2, col4);
89      get.addColumn(fam2, col5);
90      this.scan = new Scan(get);
91  
92      rowComparator = KeyValue.COMPARATOR;
93  
94    }
95  
96    private void _testMatch_ExplicitColumns(Scan scan, List<MatchCode> expected) throws IOException {
97      long now = EnvironmentEdgeManager.currentTime();
98      // 2,4,5
99      ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(this.conf, fam2,
100         0, 1, ttl, KeepDeletedCells.FALSE, 0, rowComparator), get.getFamilyMap().get(fam2),
101         now - ttl, now);
102 
103     List<KeyValue> memstore = new ArrayList<KeyValue>();
104     memstore.add(new KeyValue(row1, fam2, col1, 1, data));
105     memstore.add(new KeyValue(row1, fam2, col2, 1, data));
106     memstore.add(new KeyValue(row1, fam2, col3, 1, data));
107     memstore.add(new KeyValue(row1, fam2, col4, 1, data));
108     memstore.add(new KeyValue(row1, fam2, col5, 1, data));
109 
110     memstore.add(new KeyValue(row2, fam1, col1, data));
111 
112     List<ScanQueryMatcher.MatchCode> actual = new ArrayList<ScanQueryMatcher.MatchCode>();
113     KeyValue k = memstore.get(0);
114     qm.setRow(k.getRowArray(), k.getRowOffset(), k.getRowLength());
115 
116     for (KeyValue kv : memstore){
117       actual.add(qm.match(kv));
118     }
119 
120     assertEquals(expected.size(), actual.size());
121     for(int i=0; i< expected.size(); i++){
122       assertEquals(expected.get(i), actual.get(i));
123       if(PRINT){
124         System.out.println("expected "+expected.get(i)+
125             ", actual " +actual.get(i));
126       }
127     }
128   }
129 
130   public void testMatch_ExplicitColumns()
131   throws IOException {
132     //Moving up from the Tracker by using Gets and List<KeyValue> instead
133     //of just byte []
134 
135     //Expected result
136     List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
137     expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
138     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
139     expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
140     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
141     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW);
142     expected.add(ScanQueryMatcher.MatchCode.DONE);
143 
144     _testMatch_ExplicitColumns(scan, expected);
145   }
146 
147   public void testMatch_Wildcard()
148   throws IOException {
149     //Moving up from the Tracker by using Gets and List<KeyValue> instead
150     //of just byte []
151 
152     //Expected result
153     List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
154     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
155     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
156     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
157     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
158     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
159     expected.add(ScanQueryMatcher.MatchCode.DONE);
160 
161     long now = EnvironmentEdgeManager.currentTime();
162     ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(this.conf, fam2,
163         0, 1, ttl, KeepDeletedCells.FALSE, 0, rowComparator), null,
164         now - ttl, now);
165 
166     List<KeyValue> memstore = new ArrayList<KeyValue>();
167     memstore.add(new KeyValue(row1, fam2, col1, 1, data));
168     memstore.add(new KeyValue(row1, fam2, col2, 1, data));
169     memstore.add(new KeyValue(row1, fam2, col3, 1, data));
170     memstore.add(new KeyValue(row1, fam2, col4, 1, data));
171     memstore.add(new KeyValue(row1, fam2, col5, 1, data));
172     memstore.add(new KeyValue(row2, fam1, col1, 1, data));
173 
174     List<ScanQueryMatcher.MatchCode> actual = new ArrayList<ScanQueryMatcher.MatchCode>();
175 
176     KeyValue k = memstore.get(0);
177     qm.setRow(k.getRowArray(), k.getRowOffset(), k.getRowLength());
178 
179     for(KeyValue kv : memstore) {
180       actual.add(qm.match(kv));
181     }
182 
183     assertEquals(expected.size(), actual.size());
184     for(int i=0; i< expected.size(); i++){
185       assertEquals(expected.get(i), actual.get(i));
186       if(PRINT){
187         System.out.println("expected "+expected.get(i)+
188             ", actual " +actual.get(i));
189       }
190     }
191   }
192 
193 
194   /**
195    * Verify that {@link ScanQueryMatcher} only skips expired KeyValue
196    * instances and does not exit early from the row (skipping
197    * later non-expired KeyValues).  This version mimics a Get with
198    * explicitly specified column qualifiers.
199    *
200    * @throws IOException
201    */
202   public void testMatch_ExpiredExplicit()
203   throws IOException {
204 
205     long testTTL = 1000;
206     MatchCode [] expected = new MatchCode[] {
207         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
208         ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL,
209         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
210         ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL,
211         ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW,
212         ScanQueryMatcher.MatchCode.DONE
213     };
214 
215     long now = EnvironmentEdgeManager.currentTime();
216     ScanQueryMatcher qm =
217       new ScanQueryMatcher(scan,
218         new ScanInfo(this.conf, fam2, 0, 1, testTTL, KeepDeletedCells.FALSE, 0,
219           rowComparator), get.getFamilyMap().get(fam2), now - testTTL, now);
220 
221     KeyValue [] kvs = new KeyValue[] {
222         new KeyValue(row1, fam2, col1, now-100, data),
223         new KeyValue(row1, fam2, col2, now-50, data),
224         new KeyValue(row1, fam2, col3, now-5000, data),
225         new KeyValue(row1, fam2, col4, now-500, data),
226         new KeyValue(row1, fam2, col5, now-10000, data),
227         new KeyValue(row2, fam1, col1, now-10, data)
228     };
229 
230     KeyValue k = kvs[0];
231     qm.setRow(k.getRowArray(), k.getRowOffset(), k.getRowLength());
232 
233     List<MatchCode> actual = new ArrayList<MatchCode>(kvs.length);
234     for (KeyValue kv : kvs) {
235       actual.add( qm.match(kv) );
236     }
237 
238     assertEquals(expected.length, actual.size());
239     for (int i=0; i<expected.length; i++) {
240       if(PRINT){
241         System.out.println("expected "+expected[i]+
242             ", actual " +actual.get(i));
243       }
244       assertEquals(expected[i], actual.get(i));
245     }
246   }
247 
248 
249   /**
250    * Verify that {@link ScanQueryMatcher} only skips expired KeyValue
251    * instances and does not exit early from the row (skipping
252    * later non-expired KeyValues).  This version mimics a Get with
253    * wildcard-inferred column qualifiers.
254    *
255    * @throws IOException
256    */
257   public void testMatch_ExpiredWildcard()
258   throws IOException {
259 
260     long testTTL = 1000;
261     MatchCode [] expected = new MatchCode[] {
262         ScanQueryMatcher.MatchCode.INCLUDE,
263         ScanQueryMatcher.MatchCode.INCLUDE,
264         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
265         ScanQueryMatcher.MatchCode.INCLUDE,
266         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
267         ScanQueryMatcher.MatchCode.DONE
268     };
269 
270     long now = EnvironmentEdgeManager.currentTime();
271     ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(this.conf, fam2,
272         0, 1, testTTL, KeepDeletedCells.FALSE, 0, rowComparator), null,
273         now - testTTL, now);
274 
275     KeyValue [] kvs = new KeyValue[] {
276         new KeyValue(row1, fam2, col1, now-100, data),
277         new KeyValue(row1, fam2, col2, now-50, data),
278         new KeyValue(row1, fam2, col3, now-5000, data),
279         new KeyValue(row1, fam2, col4, now-500, data),
280         new KeyValue(row1, fam2, col5, now-10000, data),
281         new KeyValue(row2, fam1, col1, now-10, data)
282     };
283     KeyValue k = kvs[0];
284     qm.setRow(k.getRowArray(), k.getRowOffset(), k.getRowLength());
285 
286     List<ScanQueryMatcher.MatchCode> actual =
287         new ArrayList<ScanQueryMatcher.MatchCode>(kvs.length);
288     for (KeyValue kv : kvs) {
289       actual.add( qm.match(kv) );
290     }
291 
292     assertEquals(expected.length, actual.size());
293     for (int i=0; i<expected.length; i++) {
294       if(PRINT){
295         System.out.println("expected "+expected[i]+
296             ", actual " +actual.get(i));
297       }
298       assertEquals(expected[i], actual.get(i));
299     }
300   }
301 
302   public void testMatch_PartialRangeDropDeletes() throws Exception {
303     // Some ranges.
304     testDropDeletes(
305         row2, row3, new byte[][] { row1, row2, row2, row3 }, INCLUDE, SKIP, SKIP, INCLUDE);
306     testDropDeletes(row2, row3, new byte[][] { row1, row1, row2 }, INCLUDE, INCLUDE, SKIP);
307     testDropDeletes(row2, row3, new byte[][] { row2, row3, row3 }, SKIP, INCLUDE, INCLUDE);
308     testDropDeletes(row1, row3, new byte[][] { row1, row2, row3 }, SKIP, SKIP, INCLUDE);
309     // Open ranges.
310     testDropDeletes(HConstants.EMPTY_START_ROW, row3,
311         new byte[][] { row1, row2, row3 }, SKIP, SKIP, INCLUDE);
312     testDropDeletes(row2, HConstants.EMPTY_END_ROW,
313         new byte[][] { row1, row2, row3 }, INCLUDE, SKIP, SKIP);
314     testDropDeletes(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
315         new byte[][] { row1, row2, row3, row3 }, SKIP, SKIP, SKIP, SKIP);
316 
317     // No KVs in range.
318     testDropDeletes(row2, row3, new byte[][] { row1, row1, row3 }, INCLUDE, INCLUDE, INCLUDE);
319     testDropDeletes(row2, row3, new byte[][] { row3, row3 }, INCLUDE, INCLUDE);
320     testDropDeletes(row2, row3, new byte[][] { row1, row1 }, INCLUDE, INCLUDE);
321   }
322 
323   private void testDropDeletes(
324       byte[] from, byte[] to, byte[][] rows, MatchCode... expected) throws IOException {
325     long now = EnvironmentEdgeManager.currentTime();
326     // Set time to purge deletes to negative value to avoid it ever happening.
327     ScanInfo scanInfo =
328       new ScanInfo(this.conf, fam2, 0, 1, ttl, KeepDeletedCells.FALSE, -1L, rowComparator);
329     NavigableSet<byte[]> cols = get.getFamilyMap().get(fam2);
330 
331     ScanQueryMatcher qm = new ScanQueryMatcher(scan, scanInfo, cols, Long.MAX_VALUE,
332         HConstants.OLDEST_TIMESTAMP, HConstants.OLDEST_TIMESTAMP, now, from, to, null);
333     List<ScanQueryMatcher.MatchCode> actual =
334         new ArrayList<ScanQueryMatcher.MatchCode>(rows.length);
335     byte[] prevRow = null;
336     for (byte[] row : rows) {
337       if (prevRow == null || !Bytes.equals(prevRow, row)) {
338         qm.setRow(row, 0, (short)row.length);
339         prevRow = row;
340       }
341       actual.add(qm.match(new KeyValue(row, fam2, null, now, Type.Delete)));
342     }
343 
344     assertEquals(expected.length, actual.size());
345     for (int i = 0; i < expected.length; i++) {
346       if (PRINT) System.out.println("expected " + expected[i] + ", actual " + actual.get(i));
347       assertEquals(expected[i], actual.get(i));
348     }
349   }
350 }
351