View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.List;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.fs.FileSystem;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.Cell;
30  import org.apache.hadoop.hbase.CellUtil;
31  import org.apache.hadoop.hbase.HBaseTestCase;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.HRegionInfo;
34  import org.apache.hadoop.hbase.HTableDescriptor;
35  import org.apache.hadoop.hbase.KeyValue;
36  import org.apache.hadoop.hbase.MetaTableAccessor;
37  import org.apache.hadoop.hbase.TableName;
38  import org.apache.hadoop.hbase.client.Delete;
39  import org.apache.hadoop.hbase.client.Durability;
40  import org.apache.hadoop.hbase.client.Put;
41  import org.apache.hadoop.hbase.client.Result;
42  import org.apache.hadoop.hbase.client.Scan;
43  import org.apache.hadoop.hbase.testclassification.MediumTests;
44  import org.apache.hadoop.hbase.util.Bytes;
45  import org.apache.hadoop.hbase.wal.WAL;
46  import org.junit.experimental.categories.Category;
47  
48  /**
49   * TestGet is a medley of tests of get all done up as a single test.
50   * This class
51   */
52  @Category(MediumTests.class)
53  public class TestGetClosestAtOrBefore extends HBaseTestCase {
54    private static final Log LOG = LogFactory.getLog(TestGetClosestAtOrBefore.class);
55  
56    private static final byte[] T00 = Bytes.toBytes("000");
57    private static final byte[] T10 = Bytes.toBytes("010");
58    private static final byte[] T11 = Bytes.toBytes("011");
59    private static final byte[] T12 = Bytes.toBytes("012");
60    private static final byte[] T20 = Bytes.toBytes("020");
61    private static final byte[] T30 = Bytes.toBytes("030");
62    private static final byte[] T31 = Bytes.toBytes("031");
63    private static final byte[] T35 = Bytes.toBytes("035");
64    private static final byte[] T40 = Bytes.toBytes("040");
65  
66  
67  
68    public void testUsingMetaAndBinary() throws IOException {
69      FileSystem filesystem = FileSystem.get(conf);
70      Path rootdir = testDir;
71      // Up flush size else we bind up when we use default catalog flush of 16k.
72      fsTableDescriptors.get(TableName.META_TABLE_NAME).setMemStoreFlushSize(64 * 1024 * 1024);
73      HRegion mr = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO,
74        rootdir, this.conf, fsTableDescriptors.get(TableName.META_TABLE_NAME));
75      try {
76      // Write rows for three tables 'A', 'B', and 'C'.
77      for (char c = 'A'; c < 'D'; c++) {
78        HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("" + c));
79        final int last = 128;
80        final int interval = 2;
81        for (int i = 0; i <= last; i += interval) {
82          HRegionInfo hri = new HRegionInfo(htd.getTableName(),
83            i == 0? HConstants.EMPTY_BYTE_ARRAY: Bytes.toBytes((byte)i),
84            i == last? HConstants.EMPTY_BYTE_ARRAY: Bytes.toBytes((byte)i + interval));
85  
86          Put put = MetaTableAccessor.makePutFromRegionInfo(hri);
87          put.setDurability(Durability.SKIP_WAL);
88          mr.put(put);
89        }
90      }
91      InternalScanner s = mr.getScanner(new Scan());
92      try {
93        List<Cell> keys = new ArrayList<Cell>();
94          while (s.next(keys)) {
95          LOG.info(keys);
96          keys.clear();
97        }
98      } finally {
99        s.close();
100     }
101     findRow(mr, 'C', 44, 44);
102     findRow(mr, 'C', 45, 44);
103     findRow(mr, 'C', 46, 46);
104     findRow(mr, 'C', 43, 42);
105     mr.flush(true);
106     findRow(mr, 'C', 44, 44);
107     findRow(mr, 'C', 45, 44);
108     findRow(mr, 'C', 46, 46);
109     findRow(mr, 'C', 43, 42);
110     // Now delete 'C' and make sure I don't get entries from 'B'.
111     byte [] firstRowInC = HRegionInfo.createRegionName(
112         TableName.valueOf("" + 'C'),
113         HConstants.EMPTY_BYTE_ARRAY, HConstants.ZEROES, false);
114     Scan scan = new Scan(firstRowInC);
115     s = mr.getScanner(scan);
116     try {
117       List<Cell> keys = new ArrayList<Cell>();
118         while (s.next(keys)) {
119         mr.delete(new Delete(CellUtil.cloneRow(keys.get(0))));
120         keys.clear();
121       }
122     } finally {
123       s.close();
124     }
125     // Assert we get null back (pass -1).
126     findRow(mr, 'C', 44, -1);
127     findRow(mr, 'C', 45, -1);
128     findRow(mr, 'C', 46, -1);
129     findRow(mr, 'C', 43, -1);
130     mr.flush(true);
131     findRow(mr, 'C', 44, -1);
132     findRow(mr, 'C', 45, -1);
133     findRow(mr, 'C', 46, -1);
134     findRow(mr, 'C', 43, -1);
135     } finally {
136       if (mr != null) {
137         try {
138           mr.close();
139         } catch (Exception e) {
140           e.printStackTrace();
141         }
142         mr.getWAL().close();
143       }
144     }
145   }
146 
147   /*
148    * @param mr
149    * @param table
150    * @param rowToFind
151    * @param answer Pass -1 if we're not to find anything.
152    * @return Row found.
153    * @throws IOException
154    */
155   private byte [] findRow(final Region mr, final char table,
156     final int rowToFind, final int answer)
157   throws IOException {
158     TableName tableb = TableName.valueOf("" + table);
159     // Find the row.
160     byte [] tofindBytes = Bytes.toBytes((short)rowToFind);
161     byte [] metaKey = HRegionInfo.createRegionName(
162         tableb, tofindBytes,
163       HConstants.NINES, false);
164     LOG.info("find=" + new String(metaKey));
165     Result r = mr.getClosestRowBefore(metaKey, HConstants.CATALOG_FAMILY);
166     if (answer == -1) {
167       assertNull(r);
168       return null;
169     }
170     assertTrue(Bytes.compareTo(Bytes.toBytes((short)answer),
171       extractRowFromMetaRow(r.getRow())) == 0);
172     return r.getRow();
173   }
174 
175   private byte [] extractRowFromMetaRow(final byte [] b) {
176     int firstDelimiter = KeyValue.getDelimiter(b, 0, b.length,
177       HConstants.DELIMITER);
178     int lastDelimiter = KeyValue.getDelimiterInReverse(b, 0, b.length,
179       HConstants.DELIMITER);
180     int length = lastDelimiter - firstDelimiter - 1;
181     byte [] row = new byte[length];
182     System.arraycopy(b, firstDelimiter + 1, row, 0, length);
183     return row;
184   }
185 
186   /**
187    * Test file of multiple deletes and with deletes as final key.
188    * @see <a href="https://issues.apache.org/jira/browse/HBASE-751">HBASE-751</a>
189    */
190   public void testGetClosestRowBefore3() throws IOException{
191     HRegion region = null;
192     byte [] c0 = COLUMNS[0];
193     byte [] c1 = COLUMNS[1];
194     try {
195       HTableDescriptor htd = createTableDescriptor(getName());
196       region = createNewHRegion(htd, null, null);
197 
198       Put p = new Put(T00);
199       p.add(c0, c0, T00);
200       region.put(p);
201 
202       p = new Put(T10);
203       p.add(c0, c0, T10);
204       region.put(p);
205 
206       p = new Put(T20);
207       p.add(c0, c0, T20);
208       region.put(p);
209 
210       Result r = region.getClosestRowBefore(T20, c0);
211       assertTrue(Bytes.equals(T20, r.getRow()));
212 
213       Delete d = new Delete(T20);
214       d.deleteColumn(c0, c0);
215       region.delete(d);
216 
217       r = region.getClosestRowBefore(T20, c0);
218       assertTrue(Bytes.equals(T10, r.getRow()));
219 
220       p = new Put(T30);
221       p.add(c0, c0, T30);
222       region.put(p);
223 
224       r = region.getClosestRowBefore(T30, c0);
225       assertTrue(Bytes.equals(T30, r.getRow()));
226 
227       d = new Delete(T30);
228       d.deleteColumn(c0, c0);
229       region.delete(d);
230 
231       r = region.getClosestRowBefore(T30, c0);
232       assertTrue(Bytes.equals(T10, r.getRow()));
233       r = region.getClosestRowBefore(T31, c0);
234       assertTrue(Bytes.equals(T10, r.getRow()));
235 
236       region.flush(true);
237 
238       // try finding "010" after flush
239       r = region.getClosestRowBefore(T30, c0);
240       assertTrue(Bytes.equals(T10, r.getRow()));
241       r = region.getClosestRowBefore(T31, c0);
242       assertTrue(Bytes.equals(T10, r.getRow()));
243 
244       // Put into a different column family.  Should make it so I still get t10
245       p = new Put(T20);
246       p.add(c1, c1, T20);
247       region.put(p);
248 
249       r = region.getClosestRowBefore(T30, c0);
250       assertTrue(Bytes.equals(T10, r.getRow()));
251       r = region.getClosestRowBefore(T31, c0);
252       assertTrue(Bytes.equals(T10, r.getRow()));
253 
254       region.flush(true);
255 
256       r = region.getClosestRowBefore(T30, c0);
257       assertTrue(Bytes.equals(T10, r.getRow()));
258       r = region.getClosestRowBefore(T31, c0);
259       assertTrue(Bytes.equals(T10, r.getRow()));
260 
261       // Now try combo of memcache and mapfiles.  Delete the t20 COLUMS[1]
262       // in memory; make sure we get back t10 again.
263       d = new Delete(T20);
264       d.deleteColumn(c1, c1);
265       region.delete(d);
266       r = region.getClosestRowBefore(T30, c0);
267       assertTrue(Bytes.equals(T10, r.getRow()));
268 
269       // Ask for a value off the end of the file.  Should return t10.
270       r = region.getClosestRowBefore(T31, c0);
271       assertTrue(Bytes.equals(T10, r.getRow()));
272       region.flush(true);
273       r = region.getClosestRowBefore(T31, c0);
274       assertTrue(Bytes.equals(T10, r.getRow()));
275 
276       // Ok.  Let the candidate come out of hfile but have delete of
277       // the candidate be in memory.
278       p = new Put(T11);
279       p.add(c0, c0, T11);
280       region.put(p);
281       d = new Delete(T10);
282       d.deleteColumn(c1, c1);
283       r = region.getClosestRowBefore(T12, c0);
284       assertTrue(Bytes.equals(T11, r.getRow()));
285     } finally {
286       if (region != null) {
287         try {
288           WAL wal = region.getWAL();
289           region.close();
290           wal.close();
291         } catch (Exception e) {
292           e.printStackTrace();
293         }
294       }
295     }
296   }
297 
298   /** For HBASE-694 */
299   public void testGetClosestRowBefore2() throws IOException{
300     Region region = null;
301     byte [] c0 = COLUMNS[0];
302     try {
303       HTableDescriptor htd = createTableDescriptor(getName());
304       region = createNewHRegion(htd, null, null);
305 
306       Put p = new Put(T10);
307       p.add(c0, c0, T10);
308       region.put(p);
309 
310       p = new Put(T30);
311       p.add(c0, c0, T30);
312       region.put(p);
313 
314       p = new Put(T40);
315       p.add(c0, c0, T40);
316       region.put(p);
317 
318       // try finding "035"
319       Result r = region.getClosestRowBefore(T35, c0);
320       assertTrue(Bytes.equals(T30, r.getRow()));
321 
322       region.flush(true);
323 
324       // try finding "035"
325       r = region.getClosestRowBefore(T35, c0);
326       assertTrue(Bytes.equals(T30, r.getRow()));
327 
328       p = new Put(T20);
329       p.add(c0, c0, T20);
330       region.put(p);
331 
332       // try finding "035"
333       r = region.getClosestRowBefore(T35, c0);
334       assertTrue(Bytes.equals(T30, r.getRow()));
335 
336       region.flush(true);
337 
338       // try finding "035"
339       r = region.getClosestRowBefore(T35, c0);
340       assertTrue(Bytes.equals(T30, r.getRow()));
341     } finally {
342       if (region != null) {
343         try {
344           WAL wal = ((HRegion)region).getWAL();
345           ((HRegion)region).close();
346           wal.close();
347         } catch (Exception e) {
348           e.printStackTrace();
349         }
350       }
351     }
352   }
353 
354 }
355