View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.client;
20  
21  import static org.junit.Assert.assertEquals;
22  
23  import java.util.ArrayList;
24  import java.util.LinkedList;
25  import java.util.List;
26  import java.util.Map.Entry;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.HBaseTestingUtility;
32  import org.apache.hadoop.hbase.HColumnDescriptor;
33  import org.apache.hadoop.hbase.HTableDescriptor;
34  import org.apache.hadoop.hbase.TableName;
35  import org.apache.hadoop.hbase.testclassification.LargeTests;
36  import org.apache.hadoop.hbase.util.Bytes;
37  import org.junit.AfterClass;
38  import org.junit.BeforeClass;
39  import org.junit.Test;
40  import org.junit.experimental.categories.Category;
41  
42  import com.google.common.collect.Maps;
43  
44  @Category(LargeTests.class)
45  public class TestSizeFailures {
46    private static final Log LOG = LogFactory.getLog(TestSizeFailures.class);
47    protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
48    private static byte [] FAMILY = Bytes.toBytes("testFamily");
49    protected static int SLAVES = 1;
50    private static TableName TABLENAME;
51    private static final int NUM_ROWS = 1000 * 1000, NUM_COLS = 10;
52  
53    @BeforeClass
54    public static void setUpBeforeClass() throws Exception {
55      // Uncomment the following lines if more verbosity is needed for
56      // debugging (see HBASE-12285 for details).
57      //((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
58      //((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
59      //((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
60      Configuration conf = TEST_UTIL.getConfiguration();
61      conf.setBoolean("hbase.table.sanity.checks", true); // ignore sanity checks in the server
62      TEST_UTIL.startMiniCluster(SLAVES);
63  
64      // Write a bunch of data
65      TABLENAME = TableName.valueOf("testSizeFailures");
66      List<byte[]> qualifiers = new ArrayList<>();
67      for (int i = 1; i <= 10; i++) {
68        qualifiers.add(Bytes.toBytes(Integer.toString(i)));
69      }
70  
71      HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
72      HTableDescriptor desc = new HTableDescriptor(TABLENAME);
73      desc.addFamily(hcd);
74      byte[][] splits = new byte[9][2];
75      for (int i = 1; i < 10; i++) {
76        int split = 48 + i;
77        splits[i - 1][0] = (byte) (split >>> 8);
78        splits[i - 1][0] = (byte) (split);
79      }
80      TEST_UTIL.getHBaseAdmin().createTable(desc, splits);
81      Connection conn = TEST_UTIL.getConnection();
82  
83      try (Table table = conn.getTable(TABLENAME)) {
84        List<Put> puts = new LinkedList<>();
85        for (int i = 0; i < NUM_ROWS; i++) {
86          Put p = new Put(Bytes.toBytes(Integer.toString(i)));
87          for (int j = 0; j < NUM_COLS; j++) {
88            byte[] value = new byte[50];
89            Bytes.random(value);
90            p.addColumn(FAMILY, Bytes.toBytes(Integer.toString(j)), value);
91          }
92          puts.add(p);
93  
94          if (puts.size() == 1000) {
95            table.batch(puts, new Object[1000]);
96            puts.clear();
97          }
98        }
99  
100       if (puts.size() > 0) {
101         table.batch(puts, new Object[puts.size()]);
102       }
103     }
104   }
105 
106   @AfterClass
107   public static void tearDownAfterClass() throws Exception {
108     TEST_UTIL.shutdownMiniCluster();
109   }
110 
111   /**
112    * Basic client side validation of HBASE-13262
113    */
114   @Test
115   public void testScannerSeesAllRecords() throws Exception {
116     Connection conn = TEST_UTIL.getConnection();
117     try (Table table = conn.getTable(TABLENAME)) {
118       Scan s = new Scan();
119       s.addFamily(FAMILY);
120       s.setMaxResultSize(-1);
121       s.setBatch(-1);
122       s.setCaching(500);
123       Entry<Long,Long> entry = sumTable(table.getScanner(s));
124       long rowsObserved = entry.getKey();
125       long entriesObserved = entry.getValue();
126 
127       // Verify that we see 1M rows and 10M cells
128       assertEquals(NUM_ROWS, rowsObserved);
129       assertEquals(NUM_ROWS * NUM_COLS, entriesObserved);
130     }
131   }
132 
133   /**
134    * Basic client side validation of HBASE-13262
135    */
136   @Test
137   public void testSmallScannerSeesAllRecords() throws Exception {
138     Connection conn = TEST_UTIL.getConnection();
139     try (Table table = conn.getTable(TABLENAME)) {
140       Scan s = new Scan();
141       s.setSmall(true);
142       s.addFamily(FAMILY);
143       s.setMaxResultSize(-1);
144       s.setBatch(-1);
145       s.setCaching(500);
146       Entry<Long,Long> entry = sumTable(table.getScanner(s));
147       long rowsObserved = entry.getKey();
148       long entriesObserved = entry.getValue();
149 
150       // Verify that we see 1M rows and 10M cells
151       assertEquals(NUM_ROWS, rowsObserved);
152       assertEquals(NUM_ROWS * NUM_COLS, entriesObserved);
153     }
154   }
155 
156   /**
157    * Count the number of rows and the number of entries from a scanner
158    *
159    * @param scanner
160    *          The Scanner
161    * @return An entry where the first item is rows observed and the second is entries observed.
162    */
163   private Entry<Long,Long> sumTable(ResultScanner scanner) {
164     long rowsObserved = 0l;
165     long entriesObserved = 0l;
166 
167     // Read all the records in the table
168     for (Result result : scanner) {
169       rowsObserved++;
170       while (result.advance()) {
171         entriesObserved++;
172       }
173     }
174     return Maps.immutableEntry(rowsObserved,entriesObserved);
175   }
176 }