View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
3    * agreements. See the NOTICE file distributed with this work for additional information regarding
4    * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
5    * "License"); you may not use this file except in compliance with the License. You may obtain a
6    * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
7    * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
8    * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
9    * for the specific language governing permissions and limitations under the License.
10   */
11  package org.apache.hadoop.hbase;
12  
13  import static org.junit.Assert.assertEquals;
14  import static org.junit.Assert.assertTrue;
15  
16  import java.io.IOException;
17  import java.util.ArrayList;
18  import java.util.List;
19  
20  import org.apache.hadoop.hbase.client.Put;
21  import org.apache.hadoop.hbase.client.Result;
22  import org.apache.hadoop.hbase.client.ResultScanner;
23  import org.apache.hadoop.hbase.client.Scan;
24  import org.apache.hadoop.hbase.client.Table;
25  import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
26  import org.apache.hadoop.hbase.client.metrics.ServerSideScanMetrics;
27  import org.apache.hadoop.hbase.filter.BinaryComparator;
28  import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
29  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
30  import org.apache.hadoop.hbase.filter.Filter;
31  import org.apache.hadoop.hbase.filter.FilterList;
32  import org.apache.hadoop.hbase.filter.FilterList.Operator;
33  import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
34  import org.apache.hadoop.hbase.filter.RowFilter;
35  import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
36  import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
37  import org.apache.hadoop.hbase.testclassification.MediumTests;
38  import org.apache.hadoop.hbase.util.Bytes;
39  import org.junit.AfterClass;
40  import org.junit.BeforeClass;
41  import org.junit.Test;
42  import org.junit.experimental.categories.Category;
43  
44  @Category(MediumTests.class)
45  public class TestServerSideScanMetricsFromClientSide {
46    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
47  
48    private static Table TABLE = null;
49  
50    /**
51     * Table configuration
52     */
53    private static TableName TABLE_NAME = TableName.valueOf("testTable");
54  
55    private static int NUM_ROWS = 10;
56    private static byte[] ROW = Bytes.toBytes("testRow");
57    private static byte[][] ROWS = HTestConst.makeNAscii(ROW, NUM_ROWS);
58  
59    // Should keep this value below 10 to keep generation of expected kv's simple. If above 10 then
60    // table/row/cf1/... will be followed by table/row/cf10/... instead of table/row/cf2/... which
61    // breaks the simple generation of expected kv's
62    private static int NUM_FAMILIES = 1;
63    private static byte[] FAMILY = Bytes.toBytes("testFamily");
64    private static byte[][] FAMILIES = HTestConst.makeNAscii(FAMILY, NUM_FAMILIES);
65  
66    private static int NUM_QUALIFIERS = 1;
67    private static byte[] QUALIFIER = Bytes.toBytes("testQualifier");
68    private static byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, NUM_QUALIFIERS);
69  
70    private static int VALUE_SIZE = 10;
71    private static byte[] VALUE = Bytes.createMaxByteArray(VALUE_SIZE);
72  
73    private static int NUM_COLS = NUM_FAMILIES * NUM_QUALIFIERS;
74  
75    // Approximation of how large the heap size of cells in our table. Should be accessed through
76    // getCellHeapSize().
77    private static long CELL_HEAP_SIZE = -1;
78  
79    @BeforeClass
80    public static void setUpBeforeClass() throws Exception {
81      TEST_UTIL.startMiniCluster(3);
82      TABLE = createTestTable(TABLE_NAME, ROWS, FAMILIES, QUALIFIERS, VALUE);
83    }
84  
85    static Table createTestTable(TableName name, byte[][] rows, byte[][] families,
86        byte[][] qualifiers, byte[] cellValue) throws IOException {
87      Table ht = TEST_UTIL.createTable(name, families);
88      List<Put> puts = createPuts(rows, families, qualifiers, cellValue);
89      ht.put(puts);
90  
91      return ht;
92    }
93  
94    @AfterClass
95    public static void tearDownAfterClass() throws Exception {
96      TEST_UTIL.shutdownMiniCluster();
97    }
98  
99    /**
100    * Make puts to put the input value into each combination of row, family, and qualifier
101    * @param rows
102    * @param families
103    * @param qualifiers
104    * @param value
105    * @return
106    * @throws IOException
107    */
108   static ArrayList<Put> createPuts(byte[][] rows, byte[][] families, byte[][] qualifiers,
109       byte[] value) throws IOException {
110     Put put;
111     ArrayList<Put> puts = new ArrayList<>();
112 
113     for (int row = 0; row < rows.length; row++) {
114       put = new Put(rows[row]);
115       for (int fam = 0; fam < families.length; fam++) {
116         for (int qual = 0; qual < qualifiers.length; qual++) {
117           KeyValue kv = new KeyValue(rows[row], families[fam], qualifiers[qual], qual, value);
118           put.add(kv);
119         }
120       }
121       puts.add(put);
122     }
123 
124     return puts;
125   }
126 
127   /**
128    * @return The approximate heap size of a cell in the test table. All cells should have
129    *         approximately the same heap size, so the value is cached to avoid repeating the
130    *         calculation
131    * @throws Exception
132    */
133   private long getCellHeapSize() throws Exception {
134     if (CELL_HEAP_SIZE == -1) {
135       // Do a partial scan that will return a single result with a single cell
136       Scan scan = new Scan();
137       scan.setMaxResultSize(1);
138       scan.setAllowPartialResults(true);
139       ResultScanner scanner = TABLE.getScanner(scan);
140 
141       Result result = scanner.next();
142 
143       assertTrue(result != null);
144       assertTrue(result.rawCells() != null);
145       assertTrue(result.rawCells().length == 1);
146 
147       CELL_HEAP_SIZE = CellUtil.estimatedHeapSizeOf(result.rawCells()[0]);
148       scanner.close();
149     }
150 
151     return CELL_HEAP_SIZE;
152   }
153 
154   @Test
155   public void testRowsSeenMetric() throws Exception {
156     // Base scan configuration
157     Scan baseScan;
158     baseScan = new Scan();
159     baseScan.setScanMetricsEnabled(true);
160     testRowsSeenMetric(baseScan);
161 
162     // Test case that only a single result will be returned per RPC to the serer
163     baseScan.setCaching(1);
164     testRowsSeenMetric(baseScan);
165 
166     // Test case that partial results are returned from the server. At most one cell will be
167     // contained in each response
168     baseScan.setMaxResultSize(1);
169     testRowsSeenMetric(baseScan);
170 
171     // Test case that size limit is set such that a few cells are returned per partial result from
172     // the server
173     baseScan.setCaching(NUM_ROWS);
174     baseScan.setMaxResultSize(getCellHeapSize() * (NUM_COLS - 1));
175     testRowsSeenMetric(baseScan);
176   }
177 
178   public void testRowsSeenMetric(Scan baseScan) throws Exception {
179     Scan scan;
180     scan = new Scan(baseScan);
181     testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY, NUM_ROWS);
182 
183     for (int i = 0; i < ROWS.length - 1; i++) {
184       scan = new Scan(baseScan);
185       scan.setStartRow(ROWS[0]);
186       scan.setStopRow(ROWS[i + 1]);
187       testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY, i + 1);
188     }
189 
190     for (int i = ROWS.length - 1; i > 0; i--) {
191       scan = new Scan(baseScan);
192       scan.setStartRow(ROWS[i - 1]);
193       scan.setStopRow(ROWS[ROWS.length - 1]);
194       testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY, ROWS.length - i);
195     }
196 
197     // The filter should filter out all rows, but we still expect to see every row.
198     Filter filter = new RowFilter(CompareOp.EQUAL, new BinaryComparator("xyz".getBytes()));
199     scan = new Scan(baseScan);
200     scan.setFilter(filter);
201     testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY, ROWS.length);
202 
203     // Filter should pass on all rows
204     SingleColumnValueFilter singleColumnValueFilter =
205         new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS[0], CompareOp.EQUAL, VALUE);
206     scan = new Scan(baseScan);
207     scan.setFilter(singleColumnValueFilter);
208     testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY, ROWS.length);
209 
210     // Filter should filter out all rows
211     singleColumnValueFilter =
212         new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS[0], CompareOp.NOT_EQUAL, VALUE);
213     scan = new Scan(baseScan);
214     scan.setFilter(singleColumnValueFilter);
215     testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY, ROWS.length);
216   }
217 
218   @Test
219   public void testRowsFilteredMetric() throws Exception {
220     // Base scan configuration
221     Scan baseScan;
222     baseScan = new Scan();
223     baseScan.setScanMetricsEnabled(true);
224 
225     // Test case where scan uses default values
226     testRowsFilteredMetric(baseScan);
227 
228     // Test case where at most one Result is retrieved per RPC
229     baseScan.setCaching(1);
230     testRowsFilteredMetric(baseScan);
231 
232     // Test case where size limit is very restrictive and partial results will be returned from
233     // server
234     baseScan.setMaxResultSize(1);
235     testRowsFilteredMetric(baseScan);
236 
237     // Test a case where max result size limits response from server to only a few cells (not all
238     // cells from the row)
239     baseScan.setCaching(NUM_ROWS);
240     baseScan.setMaxResultSize(getCellHeapSize() * (NUM_COLS - 1));
241     testRowsSeenMetric(baseScan);
242   }
243 
244   public void testRowsFilteredMetric(Scan baseScan) throws Exception {
245     testRowsFilteredMetric(baseScan, null, 0);
246 
247     // Row filter doesn't match any row key. All rows should be filtered
248     Filter filter = new RowFilter(CompareOp.EQUAL, new BinaryComparator("xyz".getBytes()));
249     testRowsFilteredMetric(baseScan, filter, ROWS.length);
250 
251     // Filter will return results containing only the first key. Number of entire rows filtered
252     // should be 0.
253     filter = new FirstKeyOnlyFilter();
254     testRowsFilteredMetric(baseScan, filter, 0);
255 
256     // Column prefix will find some matching qualifier on each row. Number of entire rows filtered
257     // should be 0
258     filter = new ColumnPrefixFilter(QUALIFIERS[0]);
259     testRowsFilteredMetric(baseScan, filter, 0);
260 
261     // Column prefix will NOT find any matching qualifier on any row. All rows should be filtered
262     filter = new ColumnPrefixFilter("xyz".getBytes());
263     testRowsFilteredMetric(baseScan, filter, ROWS.length);
264 
265     // Matching column value should exist in each row. No rows should be filtered.
266     filter = new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS[0], CompareOp.EQUAL, VALUE);
267     testRowsFilteredMetric(baseScan, filter, 0);
268 
269     // No matching column value should exist in any row. Filter all rows
270     filter = new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS[0], CompareOp.NOT_EQUAL, VALUE);
271     testRowsFilteredMetric(baseScan, filter, ROWS.length);
272 
273     List<Filter> filters = new ArrayList<Filter>();
274     filters.add(new RowFilter(CompareOp.EQUAL, new BinaryComparator(ROWS[0])));
275     filters.add(new RowFilter(CompareOp.EQUAL, new BinaryComparator(ROWS[3])));
276     int numberOfMatchingRowFilters = filters.size();
277     filter = new FilterList(Operator.MUST_PASS_ONE, filters);
278     testRowsFilteredMetric(baseScan, filter, ROWS.length - numberOfMatchingRowFilters);
279     filters.clear();
280 
281     // Add a single column value exclude filter for each column... The net effect is that all
282     // columns will be excluded when scanning on the server side. This will result in an empty cell
283     // array in RegionScanner#nextInternal which should be interpreted as a row being filtered.
284     for (int family = 0; family < FAMILIES.length; family++) {
285       for (int qualifier = 0; qualifier < QUALIFIERS.length; qualifier++) {
286         filters.add(new SingleColumnValueExcludeFilter(FAMILIES[family], QUALIFIERS[qualifier],
287             CompareOp.EQUAL, VALUE));
288       }
289     }
290     filter = new FilterList(Operator.MUST_PASS_ONE, filters);
291     testRowsFilteredMetric(baseScan, filter, ROWS.length);
292   }
293 
294   public void testRowsFilteredMetric(Scan baseScan, Filter filter, int expectedNumFiltered)
295       throws Exception {
296     Scan scan = new Scan(baseScan);
297     if (filter != null) scan.setFilter(filter);
298     testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_FILTERED_KEY, expectedNumFiltered);
299   }
300 
301   /**
302    * Run the scan to completetion and check the metric against the specified value
303    * @param scan
304    * @param metricKey
305    * @param expectedValue
306    * @throws Exception
307    */
308   public void testMetric(Scan scan, String metricKey, long expectedValue) throws Exception {
309     assertTrue("Scan should be configured to record metrics", scan.isScanMetricsEnabled());
310     ResultScanner scanner = TABLE.getScanner(scan);
311 
312     // Iterate through all the results
313     for (Result r : scanner) {
314     }
315     scanner.close();
316     ScanMetrics metrics = scan.getScanMetrics();
317     assertTrue("Metrics are null", metrics != null);
318     assertTrue("Metric : " + metricKey + " does not exist", metrics.hasCounter(metricKey));
319     final long actualMetricValue = metrics.getCounter(metricKey).get();
320     assertEquals("Metric: " + metricKey + " Expected: " + expectedValue + " Actual: "
321         + actualMetricValue, expectedValue, actualMetricValue);
322 
323   }
324 }