View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase;
20  
21  import static org.junit.Assert.*;
22  
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.List;
26  import java.util.NavigableMap;
27  import java.util.TreeMap;
28  
29  import org.apache.hadoop.hbase.KeyValue.Type;
30  import org.apache.hadoop.hbase.testclassification.SmallTests;
31  import org.apache.hadoop.hbase.util.Bytes;
32  import org.junit.Assert;
33  import org.junit.Test;
34  import org.junit.experimental.categories.Category;
35  
36  @Category(SmallTests.class)
37  public class TestCellUtil {
38    /**
39     * CellScannable used in test. Returns a {@link TestCellScanner}
40     */
41    private class TestCellScannable implements CellScannable {
42      private final int cellsCount;
43      TestCellScannable(final int cellsCount) {
44        this.cellsCount = cellsCount;
45      }
46      @Override
47      public CellScanner cellScanner() {
48        return new TestCellScanner(this.cellsCount);
49      }
50    };
51  
52    /**
53     * CellScanner used in test.
54     */
55    private class TestCellScanner implements CellScanner {
56      private int count = 0;
57      private Cell current = null;
58      private final int cellsCount;
59  
60      TestCellScanner(final int cellsCount) {
61        this.cellsCount = cellsCount;
62      }
63  
64      @Override
65      public Cell current() {
66        return this.current;
67      }
68  
69      @Override
70      public boolean advance() throws IOException {
71        if (this.count < cellsCount) {
72          this.current = new TestCell(this.count);
73          this.count++;
74          return true;
75        }
76        return false;
77      }
78    }
79  
80    /**
81     * Cell used in test. Has row only.
82     */
83    private class TestCell implements Cell {
84      private final byte [] row;
85  
86      TestCell(final int i) {
87        this.row = Bytes.toBytes(i);
88      }
89  
90      @Override
91      public byte[] getRowArray() {
92        return this.row;
93      }
94  
95      @Override
96      public int getRowOffset() {
97        return 0;
98      }
99  
100     @Override
101     public short getRowLength() {
102       return (short)this.row.length;
103     }
104 
105     @Override
106     public byte[] getFamilyArray() {
107       // TODO Auto-generated method stub
108       return null;
109     }
110 
111     @Override
112     public int getFamilyOffset() {
113       // TODO Auto-generated method stub
114       return 0;
115     }
116 
117     @Override
118     public byte getFamilyLength() {
119       // TODO Auto-generated method stub
120       return 0;
121     }
122 
123     @Override
124     public byte[] getQualifierArray() {
125       // TODO Auto-generated method stub
126       return null;
127     }
128 
129     @Override
130     public int getQualifierOffset() {
131       // TODO Auto-generated method stub
132       return 0;
133     }
134 
135     @Override
136     public int getQualifierLength() {
137       // TODO Auto-generated method stub
138       return 0;
139     }
140 
141     @Override
142     public long getTimestamp() {
143       // TODO Auto-generated method stub
144       return 0;
145     }
146 
147     @Override
148     public byte getTypeByte() {
149       // TODO Auto-generated method stub
150       return 0;
151     }
152 
153     @Override
154     public long getMvccVersion() {
155       // TODO Auto-generated method stub
156       return 0;
157     }
158 
159     @Override
160     public byte[] getValueArray() {
161       // TODO Auto-generated method stub
162       return null;
163     }
164 
165     @Override
166     public int getValueOffset() {
167       // TODO Auto-generated method stub
168       return 0;
169     }
170 
171     @Override
172     public int getValueLength() {
173       // TODO Auto-generated method stub
174       return 0;
175     }
176 
177     @Override
178     public byte[] getTagsArray() {
179       // TODO Auto-generated method stub
180       return null;
181     }
182 
183     @Override
184     public int getTagsOffset() {
185       // TODO Auto-generated method stub
186       return 0;
187     }
188 
189     @Override
190     public byte[] getValue() {
191       // TODO Auto-generated method stub
192       return null;
193     }
194 
195     @Override
196     public byte[] getFamily() {
197       // TODO Auto-generated method stub
198       return null;
199     }
200 
201     @Override
202     public byte[] getQualifier() {
203       // TODO Auto-generated method stub
204       return null;
205     }
206 
207     @Override
208     public byte[] getRow() {
209       // TODO Auto-generated method stub
210       return null;
211     }
212 
213     @Override
214     public long getSequenceId() {
215       // TODO Auto-generated method stub
216       return 0;
217     }
218 
219     @Override
220     public int getTagsLength() {
221       // TODO Auto-generated method stub
222       return 0;
223     }
224   };
225 
226   /**
227    * Was overflowing if 100k or so lists of cellscanners to return.
228    * @throws IOException
229    */
230   @Test
231   public void testCreateCellScannerOverflow() throws IOException {
232     consume(doCreateCellScanner(1, 1), 1 * 1);
233     consume(doCreateCellScanner(3, 0), 3 * 0);
234     consume(doCreateCellScanner(3, 3), 3 * 3);
235     consume(doCreateCellScanner(0, 1), 0 * 1);
236     // Do big number. See HBASE-11813 for why.
237     final int hundredK = 100000;
238     consume(doCreateCellScanner(hundredK, 0), hundredK * 0);
239     consume(doCreateCellArray(1), 1);
240     consume(doCreateCellArray(0), 0);
241     consume(doCreateCellArray(3), 3);
242     List<CellScannable> cells = new ArrayList<CellScannable>(hundredK);
243     for (int i = 0; i < hundredK; i++) {
244       cells.add(new TestCellScannable(1));
245     }
246     consume(CellUtil.createCellScanner(cells), hundredK * 1);
247     NavigableMap<byte [], List<Cell>> m = new TreeMap<byte [], List<Cell>>(Bytes.BYTES_COMPARATOR);
248     List<Cell> cellArray = new ArrayList<Cell>(hundredK);
249     for (int i = 0; i < hundredK; i++) cellArray.add(new TestCell(i));
250     m.put(new byte [] {'f'}, cellArray);
251     consume(CellUtil.createCellScanner(m), hundredK * 1);
252   }
253 
254   private CellScanner doCreateCellArray(final int itemsPerList) {
255     Cell [] cells = new Cell [itemsPerList];
256     for (int i = 0; i < itemsPerList; i++) {
257       cells[i] = new TestCell(i);
258     }
259     return CellUtil.createCellScanner(cells);
260   }
261 
262   private CellScanner doCreateCellScanner(final int listsCount, final int itemsPerList)
263   throws IOException {
264     List<CellScannable> cells = new ArrayList<CellScannable>(listsCount);
265     for (int i = 0; i < listsCount; i++) {
266       CellScannable cs = new CellScannable() {
267         @Override
268         public CellScanner cellScanner() {
269           return new TestCellScanner(itemsPerList);
270         }
271       };
272       cells.add(cs);
273     }
274     return CellUtil.createCellScanner(cells);
275   }
276 
277   private void consume(final CellScanner scanner, final int expected) throws IOException {
278     int count = 0;
279     while (scanner.advance()) count++;
280     Assert.assertEquals(expected, count);
281   }
282 
283   @Test
284   public void testOverlappingKeys() {
285     byte[] empty = HConstants.EMPTY_BYTE_ARRAY;
286     byte[] a = Bytes.toBytes("a");
287     byte[] b = Bytes.toBytes("b");
288     byte[] c = Bytes.toBytes("c");
289     byte[] d = Bytes.toBytes("d");
290 
291     // overlaps
292     Assert.assertTrue(CellUtil.overlappingKeys(a, b, a, b));
293     Assert.assertTrue(CellUtil.overlappingKeys(a, c, a, b));
294     Assert.assertTrue(CellUtil.overlappingKeys(a, b, a, c));
295     Assert.assertTrue(CellUtil.overlappingKeys(b, c, a, c));
296     Assert.assertTrue(CellUtil.overlappingKeys(a, c, b, c));
297     Assert.assertTrue(CellUtil.overlappingKeys(a, d, b, c));
298     Assert.assertTrue(CellUtil.overlappingKeys(b, c, a, d));
299 
300     Assert.assertTrue(CellUtil.overlappingKeys(empty, b, a, b));
301     Assert.assertTrue(CellUtil.overlappingKeys(empty, b, a, c));
302 
303     Assert.assertTrue(CellUtil.overlappingKeys(a, b, empty, b));
304     Assert.assertTrue(CellUtil.overlappingKeys(a, b, empty, c));
305 
306     Assert.assertTrue(CellUtil.overlappingKeys(a, empty, a, b));
307     Assert.assertTrue(CellUtil.overlappingKeys(a, empty, a, c));
308 
309     Assert.assertTrue(CellUtil.overlappingKeys(a, b, empty, empty));
310     Assert.assertTrue(CellUtil.overlappingKeys(empty, empty, a, b));
311 
312     // non overlaps
313     Assert.assertFalse(CellUtil.overlappingKeys(a, b, c, d));
314     Assert.assertFalse(CellUtil.overlappingKeys(c, d, a, b));
315 
316     Assert.assertFalse(CellUtil.overlappingKeys(b, c, c, d));
317     Assert.assertFalse(CellUtil.overlappingKeys(b, c, c, empty));
318     Assert.assertFalse(CellUtil.overlappingKeys(b, c, d, empty));
319     Assert.assertFalse(CellUtil.overlappingKeys(c, d, b, c));
320     Assert.assertFalse(CellUtil.overlappingKeys(c, empty, b, c));
321     Assert.assertFalse(CellUtil.overlappingKeys(d, empty, b, c));
322 
323     Assert.assertFalse(CellUtil.overlappingKeys(b, c, a, b));
324     Assert.assertFalse(CellUtil.overlappingKeys(b, c, empty, b));
325     Assert.assertFalse(CellUtil.overlappingKeys(b, c, empty, a));
326     Assert.assertFalse(CellUtil.overlappingKeys(a,b, b, c));
327     Assert.assertFalse(CellUtil.overlappingKeys(empty, b, b, c));
328     Assert.assertFalse(CellUtil.overlappingKeys(empty, a, b, c));
329   }
330 
331   @Test
332   public void testFindCommonPrefixInFlatKey() {
333     // The whole key matching case
334     KeyValue kv1 = new KeyValue("r1".getBytes(), "f1".getBytes(), "q1".getBytes(), null);
335     Assert.assertEquals(kv1.getKeyLength(),
336         CellUtil.findCommonPrefixInFlatKey(kv1, kv1, true, true));
337     Assert.assertEquals(kv1.getKeyLength(),
338         CellUtil.findCommonPrefixInFlatKey(kv1, kv1, false, true));
339     Assert.assertEquals(kv1.getKeyLength() - KeyValue.TIMESTAMP_TYPE_SIZE,
340         CellUtil.findCommonPrefixInFlatKey(kv1, kv1, true, false));
341     // The rk length itself mismatch
342     KeyValue kv2 = new KeyValue("r12".getBytes(), "f1".getBytes(), "q1".getBytes(), null);
343     Assert.assertEquals(1, CellUtil.findCommonPrefixInFlatKey(kv1, kv2, true, true));
344     // part of rk is same
345     KeyValue kv3 = new KeyValue("r14".getBytes(), "f1".getBytes(), "q1".getBytes(), null);
346     Assert.assertEquals(KeyValue.ROW_LENGTH_SIZE + "r1".getBytes().length,
347         CellUtil.findCommonPrefixInFlatKey(kv2, kv3, true, true));
348     // entire rk is same but different cf name
349     KeyValue kv4 = new KeyValue("r14".getBytes(), "f2".getBytes(), "q1".getBytes(), null);
350     Assert.assertEquals(KeyValue.ROW_LENGTH_SIZE + kv3.getRowLength() + KeyValue.FAMILY_LENGTH_SIZE
351         + "f".getBytes().length, CellUtil.findCommonPrefixInFlatKey(kv3, kv4, false, true));
352     // rk and family are same and part of qualifier
353     KeyValue kv5 = new KeyValue("r14".getBytes(), "f2".getBytes(), "q123".getBytes(), null);
354     Assert.assertEquals(KeyValue.ROW_LENGTH_SIZE + kv3.getRowLength() + KeyValue.FAMILY_LENGTH_SIZE
355         + kv4.getFamilyLength() + kv4.getQualifierLength(),
356         CellUtil.findCommonPrefixInFlatKey(kv4, kv5, true, true));
357     // rk, cf and q are same. ts differs
358     KeyValue kv6 = new KeyValue("rk".getBytes(), 1234L);
359     KeyValue kv7 = new KeyValue("rk".getBytes(), 1235L);
360     // only last byte out of 8 ts bytes in ts part differs
361     Assert.assertEquals(KeyValue.ROW_LENGTH_SIZE + kv6.getRowLength() + KeyValue.FAMILY_LENGTH_SIZE
362         + kv6.getFamilyLength() + kv6.getQualifierLength() + 7,
363         CellUtil.findCommonPrefixInFlatKey(kv6, kv7, true, true));
364     // rk, cf, q and ts are same. Only type differs
365     KeyValue kv8 = new KeyValue("rk".getBytes(), 1234L, Type.Delete);
366     Assert.assertEquals(KeyValue.ROW_LENGTH_SIZE + kv6.getRowLength() + KeyValue.FAMILY_LENGTH_SIZE
367         + kv6.getFamilyLength() + kv6.getQualifierLength() + KeyValue.TIMESTAMP_SIZE,
368         CellUtil.findCommonPrefixInFlatKey(kv6, kv8, true, true));
369     // With out TS_TYPE check
370     Assert.assertEquals(KeyValue.ROW_LENGTH_SIZE + kv6.getRowLength() + KeyValue.FAMILY_LENGTH_SIZE
371         + kv6.getFamilyLength() + kv6.getQualifierLength(),
372         CellUtil.findCommonPrefixInFlatKey(kv6, kv8, true, false));
373   }
374 
375   /**
376    * Assert CellUtil makes Cell toStrings same way we do KeyValue toStrings.
377    */
378   @Test
379   public void testToString() {
380     byte [] row = Bytes.toBytes("row");
381     long ts = 123l;
382     // Make a KeyValue and a Cell and see if same toString result.
383     KeyValue kv = new KeyValue(row, HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY,
384         ts, KeyValue.Type.Minimum, HConstants.EMPTY_BYTE_ARRAY);
385     Cell cell = CellUtil.createCell(row, HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY,
386         ts, KeyValue.Type.Minimum.getCode(), HConstants.EMPTY_BYTE_ARRAY);
387     String cellToString = CellUtil.getCellKeyAsString(cell);
388     assertEquals(kv.toString(), cellToString);
389     // Do another w/ non-null family.
390     byte [] f = new byte [] {'f'};
391     byte [] q = new byte [] {'q'};
392     kv = new KeyValue(row, f, q, ts, KeyValue.Type.Minimum, HConstants.EMPTY_BYTE_ARRAY);
393     cell = CellUtil.createCell(row, f, q, ts, KeyValue.Type.Minimum.getCode(),
394         HConstants.EMPTY_BYTE_ARRAY);
395     cellToString = CellUtil.getCellKeyAsString(cell);
396     assertEquals(kv.toString(), cellToString);
397 
398   }
399 
400   @Test
401   public void testToString1() {
402     String row = "test.row";
403     String family = "test.family";
404     String qualifier = "test.qualifier";
405     long timestamp = 42;
406     Type type = Type.Put;
407     String value = "test.value";
408     long seqId = 1042;
409 
410     Cell cell = CellUtil.createCell(Bytes.toBytes(row), Bytes.toBytes(family),
411       Bytes.toBytes(qualifier), timestamp, type.getCode(), Bytes.toBytes(value), seqId);
412 
413     String nonVerbose = CellUtil.toString(cell, false);
414     String verbose = CellUtil.toString(cell, true);
415 
416     System.out.println("nonVerbose=" + nonVerbose);
417     System.out.println("verbose=" + verbose);
418 
419     Assert.assertEquals(
420         String.format("%s/%s:%s/%d/%s/vlen=%s/seqid=%s",
421           row, family, qualifier, timestamp, type.toString(),
422           Bytes.toBytes(value).length, seqId),
423         nonVerbose);
424 
425     Assert.assertEquals(
426       String.format("%s/%s:%s/%d/%s/vlen=%s/seqid=%s/%s",
427         row, family, qualifier, timestamp, type.toString(), Bytes.toBytes(value).length,
428         seqId, value),
429       verbose);
430 
431     // TODO: test with tags
432   }
433 }