View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   * http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.client;
20  
21  import org.apache.hadoop.hbase.CompatibilityFactory;
22  import org.apache.hadoop.hbase.HBaseTestingUtility;
23  import org.apache.hadoop.hbase.HColumnDescriptor;
24  import org.apache.hadoop.hbase.HConstants;
25  import org.apache.hadoop.hbase.HTableDescriptor;
26  import org.apache.hadoop.hbase.TableName;
27  import org.apache.hadoop.hbase.Waiter;
28  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
29  import org.apache.hadoop.hbase.ipc.RpcServerInterface;
30  import org.apache.hadoop.hbase.metrics.BaseSource;
31  import org.apache.hadoop.hbase.regionserver.HRegionServer;
32  import org.apache.hadoop.hbase.test.MetricsAssertHelper;
33  import org.apache.hadoop.hbase.testclassification.ClientTests;
34  import org.apache.hadoop.hbase.testclassification.MediumTests;
35  import org.apache.hadoop.hbase.util.Bytes;
36  import org.junit.AfterClass;
37  import org.junit.BeforeClass;
38  import org.junit.Test;
39  import org.junit.experimental.categories.Category;
40  
41  import java.util.ArrayList;
42  import java.util.List;
43  import java.util.concurrent.ThreadLocalRandom;
44  
45  import static junit.framework.TestCase.assertEquals;
46  
47  /**
48   * This test sets the multi size WAAAAAY low and then checks to make sure that gets will still make
49   * progress.
50   */
51  @Category({MediumTests.class, ClientTests.class})
52  public class TestMultiRespectsLimits {
53    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
54    private static final MetricsAssertHelper METRICS_ASSERT =
55        CompatibilityFactory.getInstance(MetricsAssertHelper.class);
56    private final static byte[] FAMILY = Bytes.toBytes("D");
57    public static final int MAX_SIZE = 500;
58  
59    @BeforeClass
60    public static void setUpBeforeClass() throws Exception {
61      TEST_UTIL.getConfiguration().setLong(
62          HConstants.HBASE_SERVER_SCANNER_MAX_RESULT_SIZE_KEY,
63          MAX_SIZE);
64  
65      // Only start on regionserver so that all regions are on the same server.
66      TEST_UTIL.startMiniCluster(1);
67    }
68  
69    @AfterClass
70    public static void tearDownAfterClass() throws Exception {
71      TEST_UTIL.shutdownMiniCluster();
72    }
73  
74    @Test
75    public void testMultiLimits() throws Exception {
76      final TableName name = TableName.valueOf("testMultiLimits");
77      Table t = TEST_UTIL.createTable(name, FAMILY);
78      TEST_UTIL.loadTable(t, FAMILY, false);
79  
80      // Split the table to make sure that the chunking happens accross regions.
81      try (final Admin admin = TEST_UTIL.getHBaseAdmin()) {
82        admin.split(name);
83        TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
84          @Override
85          public boolean evaluate() throws Exception {
86            return admin.getTableRegions(name).size() > 1;
87          }
88        });
89      }
90      List<Get> gets = new ArrayList<>(MAX_SIZE);
91  
92      for (int i = 0; i < MAX_SIZE; i++) {
93        gets.add(new Get(HBaseTestingUtility.ROWS[i]));
94      }
95  
96      RpcServerInterface rpcServer = TEST_UTIL.getHBaseCluster().getRegionServer(0).getRpcServer();
97      BaseSource s = rpcServer.getMetrics().getMetricsSource();
98      long startingExceptions = METRICS_ASSERT.getCounter("exceptions", s);
99      long startingMultiExceptions = METRICS_ASSERT.getCounter("exceptions.multiResponseTooLarge", s);
100 
101     Result[] results = t.get(gets);
102     assertEquals(MAX_SIZE, results.length);
103 
104     // Cells from TEST_UTIL.loadTable have a length of 27.
105     // Multiplying by less than that gives an easy lower bound on size.
106     // However in reality each kv is being reported as much higher than that.
107     METRICS_ASSERT.assertCounterGt("exceptions",
108         startingExceptions + ((MAX_SIZE * 25) / MAX_SIZE), s);
109     METRICS_ASSERT.assertCounterGt("exceptions.multiResponseTooLarge",
110         startingMultiExceptions + ((MAX_SIZE * 25) / MAX_SIZE), s);
111   }
112 
113   @Test
114   public void testBlockMultiLimits() throws Exception {
115     final TableName name = TableName.valueOf("testBlockMultiLimits");
116     HTableDescriptor desc = new HTableDescriptor(name);
117     HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
118     hcd.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
119     desc.addFamily(hcd);
120     TEST_UTIL.getHBaseAdmin().createTable(desc);
121     Table t = TEST_UTIL.getConnection().getTable(name);
122 
123     final HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(0);
124     RpcServerInterface rpcServer = regionServer.getRpcServer();
125     BaseSource s = rpcServer.getMetrics().getMetricsSource();
126     long startingExceptions = METRICS_ASSERT.getCounter("exceptions", s);
127     long startingMultiExceptions = METRICS_ASSERT.getCounter("exceptions.multiResponseTooLarge", s);
128 
129     byte[] row = Bytes.toBytes("TEST");
130     byte[][] cols = new byte[][]{
131         Bytes.toBytes("0"), // Get this
132         Bytes.toBytes("1"), // Buffer
133         Bytes.toBytes("2"), // Buffer
134         Bytes.toBytes("3"), // Get This
135         Bytes.toBytes("4"), // Buffer
136         Bytes.toBytes("5"), // Buffer
137     };
138 
139     // Set the value size so that one result will be less than the MAX_SIE
140     // however the block being reference will be larger than MAX_SIZE.
141     // This should cause the regionserver to try and send a result immediately.
142     byte[] value = new byte[MAX_SIZE - 100];
143     ThreadLocalRandom.current().nextBytes(value);
144 
145     for (byte[] col:cols) {
146       Put p = new Put(row);
147       p.addImmutable(FAMILY, col, value);
148       t.put(p);
149     }
150 
151     // Make sure that a flush happens
152     try (final Admin admin = TEST_UTIL.getHBaseAdmin()) {
153       admin.flush(name);
154       TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
155         @Override
156         public boolean evaluate() throws Exception {
157           return regionServer.getOnlineRegions(name).get(0).getMaxFlushedSeqId() > 3;
158         }
159       });
160     }
161 
162     List<Get> gets = new ArrayList<>(2);
163     Get g0 = new Get(row);
164     g0.addColumn(FAMILY, cols[0]);
165     gets.add(g0);
166 
167     Get g2 = new Get(row);
168     g2.addColumn(FAMILY, cols[3]);
169     gets.add(g2);
170 
171     Result[] results = t.get(gets);
172     assertEquals(2, results.length);
173     METRICS_ASSERT.assertCounterGt("exceptions", startingExceptions, s);
174     METRICS_ASSERT.assertCounterGt("exceptions.multiResponseTooLarge",
175         startingMultiExceptions, s);
176   }
177 }