1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase;
19
20 import static org.junit.Assert.assertArrayEquals;
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.util.ArrayList;
28 import java.util.LinkedHashSet;
29 import java.util.List;
30 import java.util.Set;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.hbase.client.ClientScanner;
35 import org.apache.hadoop.hbase.client.Delete;
36 import org.apache.hadoop.hbase.client.Put;
37 import org.apache.hadoop.hbase.client.Result;
38 import org.apache.hadoop.hbase.client.ResultScanner;
39 import org.apache.hadoop.hbase.client.Scan;
40 import org.apache.hadoop.hbase.client.Table;
41 import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
42 import org.apache.hadoop.hbase.filter.ColumnRangeFilter;
43 import org.apache.hadoop.hbase.filter.Filter;
44 import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
45 import org.apache.hadoop.hbase.filter.FirstKeyValueMatchingQualifiersFilter;
46 import org.apache.hadoop.hbase.filter.RandomRowFilter;
47 import org.apache.hadoop.hbase.testclassification.MediumTests;
48 import org.apache.hadoop.hbase.util.Bytes;
49 import org.apache.hadoop.hbase.util.Pair;
50 import org.junit.AfterClass;
51 import org.junit.BeforeClass;
52 import org.junit.Test;
53 import org.junit.experimental.categories.Category;
54
55
56
57
58
59
60
61
62
63
64
65 @Category(MediumTests.class)
66 public class TestPartialResultsFromClientSide {
67 private static final Log LOG = LogFactory.getLog(TestPartialResultsFromClientSide.class);
68
69 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
70 private final static int MINICLUSTER_SIZE = 5;
71 private static Table TABLE = null;
72
73
74
75
76 private static TableName TABLE_NAME = TableName.valueOf("testTable");
77
78 private static int NUM_ROWS = 5;
79 private static byte[] ROW = Bytes.toBytes("testRow");
80 private static byte[][] ROWS = HTestConst.makeNAscii(ROW, NUM_ROWS);
81
82
83
84
85 private static int NUM_FAMILIES = 10;
86 private static byte[] FAMILY = Bytes.toBytes("testFamily");
87 private static byte[][] FAMILIES = HTestConst.makeNAscii(FAMILY, NUM_FAMILIES);
88
89 private static int NUM_QUALIFIERS = 10;
90 private static byte[] QUALIFIER = Bytes.toBytes("testQualifier");
91 private static byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, NUM_QUALIFIERS);
92
93 private static int VALUE_SIZE = 1024;
94 private static byte[] VALUE = Bytes.createMaxByteArray(VALUE_SIZE);
95
96 private static int NUM_COLS = NUM_FAMILIES * NUM_QUALIFIERS;
97
98
99
100 private static long CELL_HEAP_SIZE = -1;
101
102 @BeforeClass
103 public static void setUpBeforeClass() throws Exception {
104 TEST_UTIL.startMiniCluster(MINICLUSTER_SIZE);
105 TEST_UTIL.getHBaseAdmin().setBalancerRunning(false, true);
106 TABLE = createTestTable(TABLE_NAME, ROWS, FAMILIES, QUALIFIERS, VALUE);
107 }
108
109 static Table createTestTable(TableName name, byte[][] rows, byte[][] families,
110 byte[][] qualifiers, byte[] cellValue) throws IOException {
111 Table ht = TEST_UTIL.createTable(name, families);
112 List<Put> puts = createPuts(rows, families, qualifiers, cellValue);
113 ht.put(puts);
114
115 return ht;
116 }
117
118 @AfterClass
119 public static void tearDownAfterClass() throws Exception {
120 TEST_UTIL.shutdownMiniCluster();
121 }
122
123
124
125
126
127
128 @Test
129 public void testExpectedValuesOfPartialResults() throws Exception {
130 testExpectedValuesOfPartialResults(false);
131 testExpectedValuesOfPartialResults(true);
132 }
133
134 public void testExpectedValuesOfPartialResults(boolean reversed) throws Exception {
135 Scan partialScan = new Scan();
136 partialScan.setMaxVersions();
137
138
139 partialScan.setMaxResultSize(1);
140 partialScan.setReversed(reversed);
141 ResultScanner partialScanner = TABLE.getScanner(partialScan);
142
143 final int startRow = reversed ? ROWS.length - 1 : 0;
144 final int endRow = reversed ? -1 : ROWS.length;
145 final int loopDelta = reversed ? -1 : 1;
146 String message;
147
148 for (int row = startRow; row != endRow; row = row + loopDelta) {
149 message = "Ensuring the expected keyValues are present for row " + row;
150 List<Cell> expectedKeyValues = createKeyValuesForRow(ROWS[row], FAMILIES, QUALIFIERS, VALUE);
151 Result result = partialScanner.next();
152 assertFalse(result.isPartial());
153 verifyResult(result, expectedKeyValues, message);
154 }
155
156 partialScanner.close();
157 }
158
159
160
161
162
163 @Test
164 public void testAllowPartialResults() throws Exception {
165 Scan scan = new Scan();
166 scan.setAllowPartialResults(true);
167 scan.setMaxResultSize(1);
168 ResultScanner scanner = TABLE.getScanner(scan);
169 Result result = scanner.next();
170
171 assertTrue(result != null);
172 assertTrue(result.isPartial());
173 assertTrue(result.rawCells() != null);
174 assertTrue(result.rawCells().length == 1);
175
176 scanner.close();
177
178 scan.setAllowPartialResults(false);
179 scanner = TABLE.getScanner(scan);
180 result = scanner.next();
181
182 assertTrue(result != null);
183 assertTrue(!result.isPartial());
184 assertTrue(result.rawCells() != null);
185 assertTrue(result.rawCells().length == NUM_COLS);
186
187 scanner.close();
188 }
189
190
191
192
193
194
195
196 @Test
197 public void testEquivalenceOfScanResults() throws Exception {
198 Scan oneShotScan = new Scan();
199 oneShotScan.setMaxResultSize(Long.MAX_VALUE);
200
201 Scan partialScan = new Scan(oneShotScan);
202 partialScan.setMaxResultSize(1);
203
204 testEquivalenceOfScanResults(TABLE, oneShotScan, partialScan);
205 }
206
207 public void testEquivalenceOfScanResults(Table table, Scan scan1, Scan scan2) throws Exception {
208 ResultScanner scanner1 = table.getScanner(scan1);
209 ResultScanner scanner2 = table.getScanner(scan2);
210
211 Result r1 = null;
212 Result r2 = null;
213 int count = 0;
214
215 while ((r1 = scanner1.next()) != null) {
216 r2 = scanner2.next();
217
218 assertTrue(r2 != null);
219 compareResults(r1, r2, "Comparing result #" + count);
220 count++;
221 }
222
223 r2 = scanner2.next();
224 assertTrue("r2: " + r2 + " Should be null", r2 == null);
225
226 scanner1.close();
227 scanner2.close();
228 }
229
230
231
232
233
234 @Test
235 public void testOrderingOfCellsInPartialResults() throws Exception {
236 Scan scan = new Scan();
237
238 for (int col = 1; col <= NUM_COLS; col++) {
239 scan.setMaxResultSize(getResultSizeForNumberOfCells(col));
240 testOrderingOfCellsInPartialResults(scan);
241
242
243 scan.setReversed(true);
244 testOrderingOfCellsInPartialResults(scan);
245 }
246 }
247
248 public void testOrderingOfCellsInPartialResults(final Scan basePartialScan) throws Exception {
249
250
251
252 Scan partialScan = new Scan(basePartialScan);
253 partialScan.setAllowPartialResults(true);
254 ResultScanner partialScanner = TABLE.getScanner(partialScan);
255
256
257 Scan oneShotScan = new Scan(basePartialScan);
258 oneShotScan.setMaxResultSize(Long.MAX_VALUE);
259 oneShotScan.setCaching(ROWS.length);
260 ResultScanner oneShotScanner = TABLE.getScanner(oneShotScan);
261
262 Result oneShotResult = oneShotScanner.next();
263 Result partialResult = null;
264 int iterationCount = 0;
265
266 while (oneShotResult != null && oneShotResult.rawCells() != null) {
267 List<Cell> aggregatePartialCells = new ArrayList<Cell>();
268 do {
269 partialResult = partialScanner.next();
270 assertTrue("Partial Result is null. iteration: " + iterationCount, partialResult != null);
271 assertTrue("Partial cells are null. iteration: " + iterationCount,
272 partialResult.rawCells() != null);
273
274 for (Cell c : partialResult.rawCells()) {
275 aggregatePartialCells.add(c);
276 }
277 } while (partialResult.isPartial());
278
279 assertTrue("Number of cells differs. iteration: " + iterationCount,
280 oneShotResult.rawCells().length == aggregatePartialCells.size());
281 final Cell[] oneShotCells = oneShotResult.rawCells();
282 for (int cell = 0; cell < oneShotCells.length; cell++) {
283 Cell oneShotCell = oneShotCells[cell];
284 Cell partialCell = aggregatePartialCells.get(cell);
285
286 assertTrue("One shot cell was null", oneShotCell != null);
287 assertTrue("Partial cell was null", partialCell != null);
288 assertTrue("Cell differs. oneShotCell:" + oneShotCell + " partialCell:" + partialCell,
289 oneShotCell.equals(partialCell));
290 }
291
292 oneShotResult = oneShotScanner.next();
293 iterationCount++;
294 }
295
296 assertTrue(partialScanner.next() == null);
297
298 partialScanner.close();
299 oneShotScanner.close();
300 }
301
302
303
304
305
306
307 @Test
308 public void testExpectedNumberOfCellsPerPartialResult() throws Exception {
309 Scan scan = new Scan();
310 testExpectedNumberOfCellsPerPartialResult(scan);
311
312 scan.setReversed(true);
313 testExpectedNumberOfCellsPerPartialResult(scan);
314 }
315
316 public void testExpectedNumberOfCellsPerPartialResult(Scan baseScan) throws Exception {
317 for (int expectedCells = 1; expectedCells <= NUM_COLS; expectedCells++) {
318 testExpectedNumberOfCellsPerPartialResult(baseScan, expectedCells);
319 }
320 }
321
322 public void testExpectedNumberOfCellsPerPartialResult(Scan baseScan, int expectedNumberOfCells)
323 throws Exception {
324
325 if (LOG.isInfoEnabled()) LOG.info("groupSize:" + expectedNumberOfCells);
326
327
328
329
330
331 Scan scan = new Scan(baseScan);
332 scan.setAllowPartialResults(true);
333 scan.setMaxResultSize(getResultSizeForNumberOfCells(expectedNumberOfCells));
334
335 ResultScanner scanner = TABLE.getScanner(scan);
336 Result result = null;
337 byte[] prevRow = null;
338 while ((result = scanner.next()) != null) {
339 assertTrue(result.rawCells() != null);
340
341
342
343
344
345 assertTrue(
346 "Result's cell count differed from expected number. result: " + result,
347 result.rawCells().length == expectedNumberOfCells || !result.isPartial()
348 || !Bytes.equals(prevRow, result.getRow()));
349 prevRow = result.getRow();
350 }
351
352 scanner.close();
353 }
354
355
356
357
358
359
360
361 private long getCellHeapSize() throws Exception {
362 if (CELL_HEAP_SIZE == -1) {
363
364 Scan scan = new Scan();
365 scan.setMaxResultSize(1);
366 scan.setAllowPartialResults(true);
367 ResultScanner scanner = TABLE.getScanner(scan);
368
369 Result result = scanner.next();
370
371 assertTrue(result != null);
372 assertTrue(result.rawCells() != null);
373 assertTrue(result.rawCells().length == 1);
374
375 CELL_HEAP_SIZE = CellUtil.estimatedHeapSizeOf(result.rawCells()[0]);
376 if (LOG.isInfoEnabled()) LOG.info("Cell heap size: " + CELL_HEAP_SIZE);
377 scanner.close();
378 }
379
380 return CELL_HEAP_SIZE;
381 }
382
383
384
385
386
387
388
389 private long getResultSizeForNumberOfCells(int numberOfCells) throws Exception {
390 return getCellHeapSize() * numberOfCells;
391 }
392
393
394
395
396 @Test
397 public void testPartialResultsAndBatch() throws Exception {
398 for (int batch = 1; batch <= NUM_COLS / 4; batch++) {
399 for (int cellsPerPartial = 1; cellsPerPartial <= NUM_COLS / 4; cellsPerPartial++) {
400 testPartialResultsAndBatch(batch, cellsPerPartial);
401 }
402 }
403 }
404
405 public void testPartialResultsAndBatch(final int batch, final int cellsPerPartialResult)
406 throws Exception {
407 if (LOG.isInfoEnabled()) {
408 LOG.info("batch: " + batch + " cellsPerPartialResult: " + cellsPerPartialResult);
409 }
410
411 Scan scan = new Scan();
412 scan.setMaxResultSize(getResultSizeForNumberOfCells(cellsPerPartialResult));
413 scan.setBatch(batch);
414 ResultScanner scanner = TABLE.getScanner(scan);
415 Result result = scanner.next();
416 int repCount = 0;
417
418 while ((result = scanner.next()) != null) {
419 assertTrue(result.rawCells() != null);
420
421 if (result.isPartial()) {
422 final String error =
423 "Cells:" + result.rawCells().length + " Batch size:" + batch
424 + " cellsPerPartialResult:" + cellsPerPartialResult + " rep:" + repCount;
425 assertTrue(error, result.rawCells().length <= Math.min(batch, cellsPerPartialResult));
426 } else {
427 assertTrue(result.rawCells().length <= batch);
428 }
429 repCount++;
430 }
431
432 scanner.close();
433 }
434
435
436
437
438
439 @Test
440 public void testPartialResultsReassembly() throws Exception {
441 Scan scan = new Scan();
442 testPartialResultsReassembly(scan);
443 scan.setReversed(true);
444 testPartialResultsReassembly(scan);
445 }
446
447 public void testPartialResultsReassembly(Scan scanBase) throws Exception {
448 Scan partialScan = new Scan(scanBase);
449 partialScan.setMaxResultSize(1);
450 partialScan.setAllowPartialResults(true);
451 ResultScanner partialScanner = TABLE.getScanner(partialScan);
452
453 Scan oneShotScan = new Scan(scanBase);
454 oneShotScan.setMaxResultSize(Long.MAX_VALUE);
455 ResultScanner oneShotScanner = TABLE.getScanner(oneShotScan);
456
457 ArrayList<Result> partials = new ArrayList<>();
458 for (int i = 0; i < NUM_ROWS; i++) {
459 Result partialResult = null;
460 Result completeResult = null;
461 Result oneShotResult = null;
462 partials.clear();
463
464 do {
465 partialResult = partialScanner.next();
466 partials.add(partialResult);
467 } while (partialResult != null && partialResult.isPartial());
468
469 completeResult = Result.createCompleteResult(partials);
470 oneShotResult = oneShotScanner.next();
471
472 compareResults(completeResult, oneShotResult, null);
473 }
474
475 assertTrue(oneShotScanner.next() == null);
476 assertTrue(partialScanner.next() == null);
477
478 oneShotScanner.close();
479 partialScanner.close();
480 }
481
482
483
484
485
486 @Test
487 public void testExceptionThrownOnMismatchedPartialResults() throws IOException {
488 assertTrue(NUM_ROWS >= 2);
489
490 ArrayList<Result> partials = new ArrayList<>();
491 Scan scan = new Scan();
492 scan.setMaxResultSize(Long.MAX_VALUE);
493 ResultScanner scanner = TABLE.getScanner(scan);
494 Result r1 = scanner.next();
495 partials.add(r1);
496 Result r2 = scanner.next();
497 partials.add(r2);
498
499 assertFalse(Bytes.equals(r1.getRow(), r2.getRow()));
500
501 try {
502 Result.createCompleteResult(partials);
503 fail("r1 and r2 are from different rows. It should not be possible to combine them into"
504 + " a single result");
505 } catch (IOException e) {
506 }
507
508 scanner.close();
509 }
510
511
512
513
514
515
516 @Test
517 public void testNoPartialResultsWhenRowFilterPresent() throws Exception {
518 Scan scan = new Scan();
519 scan.setMaxResultSize(1);
520 scan.setAllowPartialResults(true);
521
522
523 scan.setFilter(new RandomRowFilter(1.0f));
524 ResultScanner scanner = TABLE.getScanner(scan);
525
526 Result r = null;
527 while ((r = scanner.next()) != null) {
528 assertFalse(r.isPartial());
529 }
530
531 scanner.close();
532 }
533
534
535
536
537
538
539
540
541 @Test
542 public void testPartialResultsAndCaching() throws Exception {
543 for (int caching = 1; caching <= NUM_ROWS; caching++) {
544 for (int maxResultRows = 0; maxResultRows <= NUM_ROWS; maxResultRows++) {
545 testPartialResultsAndCaching(maxResultRows, caching);
546 }
547 }
548 }
549
550
551
552
553
554
555 public void testPartialResultsAndCaching(int resultSizeRowLimit, int cachingRowLimit)
556 throws Exception {
557 Scan scan = new Scan();
558 scan.setAllowPartialResults(true);
559
560
561
562
563
564 int cellOffset = NUM_COLS / 3;
565 long maxResultSize = getResultSizeForNumberOfCells(resultSizeRowLimit * NUM_COLS + cellOffset);
566 scan.setMaxResultSize(maxResultSize);
567 scan.setCaching(cachingRowLimit);
568
569 ResultScanner scanner = TABLE.getScanner(scan);
570 ClientScanner clientScanner = (ClientScanner) scanner;
571 Result r = null;
572
573
574
575
576 boolean expectToSeePartialResults = resultSizeRowLimit < cachingRowLimit;
577 while ((r = clientScanner.next()) != null) {
578 assertTrue(!r.isPartial() || expectToSeePartialResults);
579 }
580
581 scanner.close();
582 }
583
584
585
586
587
588
589
590
591 @Test
592 public void testSmallScansDoNotAllowPartials() throws Exception {
593 Scan scan = new Scan();
594 testSmallScansDoNotAllowPartials(scan);
595 scan.setReversed(true);
596 testSmallScansDoNotAllowPartials(scan);
597 }
598
599 public void testSmallScansDoNotAllowPartials(Scan baseScan) throws Exception {
600 Scan scan = new Scan(baseScan);
601 scan.setAllowPartialResults(true);
602 scan.setSmall(true);
603 scan.setMaxResultSize(1);
604
605 ResultScanner scanner = TABLE.getScanner(scan);
606 Result r = null;
607
608 while ((r = scanner.next()) != null) {
609 assertFalse(r.isPartial());
610 }
611
612 scanner.close();
613 }
614
615
616
617
618
619
620
621
622
623
624 static ArrayList<Put> createPuts(byte[][] rows, byte[][] families, byte[][] qualifiers,
625 byte[] value) throws IOException {
626 Put put;
627 ArrayList<Put> puts = new ArrayList<>();
628
629 for (int row = 0; row < rows.length; row++) {
630 put = new Put(rows[row]);
631 for (int fam = 0; fam < families.length; fam++) {
632 for (int qual = 0; qual < qualifiers.length; qual++) {
633 KeyValue kv = new KeyValue(rows[row], families[fam], qualifiers[qual], qual, value);
634 put.add(kv);
635 }
636 }
637 puts.add(put);
638 }
639
640 return puts;
641 }
642
643
644
645
646
647
648
649
650
651
652 static ArrayList<Cell> createKeyValuesForRow(byte[] row, byte[][] families, byte[][] qualifiers,
653 byte[] value) {
654 ArrayList<Cell> outList = new ArrayList<>();
655 for (int fam = 0; fam < families.length; fam++) {
656 for (int qual = 0; qual < qualifiers.length; qual++) {
657 outList.add(new KeyValue(row, families[fam], qualifiers[qual], qual, value));
658 }
659 }
660 return outList;
661 }
662
663
664
665
666
667
668
669 static void verifyResult(Result result, List<Cell> expKvList, String msg) {
670 if (LOG.isInfoEnabled()) {
671 LOG.info(msg);
672 LOG.info("Expected count: " + expKvList.size());
673 LOG.info("Actual count: " + result.size());
674 }
675
676 if (expKvList.size() == 0) return;
677
678 int i = 0;
679 for (Cell kv : result.rawCells()) {
680 if (i >= expKvList.size()) {
681 break;
682 }
683
684 Cell kvExp = expKvList.get(i++);
685 assertTrue("Not equal. get kv: " + kv.toString() + " exp kv: " + kvExp.toString(),
686 kvExp.equals(kv));
687 }
688
689 assertEquals(expKvList.size(), result.size());
690 }
691
692
693
694
695
696
697
698 static void compareResults(Result r1, Result r2, final String message) {
699 if (LOG.isInfoEnabled()) {
700 if (message != null) LOG.info(message);
701 LOG.info("r1: " + r1);
702 LOG.info("r2: " + r2);
703 }
704
705 final String failureMessage = "Results r1:" + r1 + " \nr2:" + r2 + " are not equivalent";
706 if (r1 == null && r2 == null) fail(failureMessage);
707 else if (r1 == null || r2 == null) fail(failureMessage);
708
709 try {
710 Result.compareResults(r1, r2);
711 } catch (Exception e) {
712 fail(failureMessage);
713 }
714 }
715
716 @Test
717 public void testReadPointAndPartialResults() throws Exception {
718 TableName testName = TableName.valueOf("testReadPointAndPartialResults");
719 int numRows = 5;
720 int numFamilies = 5;
721 int numQualifiers = 5;
722 byte[][] rows = HTestConst.makeNAscii(Bytes.toBytes("testRow"), numRows);
723 byte[][] families = HTestConst.makeNAscii(Bytes.toBytes("testFamily"), numFamilies);
724 byte[][] qualifiers = HTestConst.makeNAscii(Bytes.toBytes("testQualifier"), numQualifiers);
725 byte[] value = Bytes.createMaxByteArray(100);
726
727 Table tmpTable = createTestTable(testName, rows, families, qualifiers, value);
728
729 Scan scan = new Scan();
730 scan.setMaxResultSize(1);
731 scan.setAllowPartialResults(true);
732
733
734 ResultScanner scanner = tmpTable.getScanner(scan);
735
736 Delete delete1 = new Delete(rows[0]);
737 delete1.addColumn(families[0], qualifiers[0], 0);
738 tmpTable.delete(delete1);
739
740 Delete delete2 = new Delete(rows[1]);
741 delete2.addColumn(families[1], qualifiers[1], 1);
742 tmpTable.delete(delete2);
743
744
745 int scannerCount = countCellsFromScanner(scanner);
746 int expectedCount = numRows * numFamilies * numQualifiers;
747 assertTrue("scannerCount: " + scannerCount + " expectedCount: " + expectedCount,
748 scannerCount == expectedCount);
749
750
751 scanner = tmpTable.getScanner(scan);
752 scannerCount = countCellsFromScanner(scanner);
753 expectedCount = numRows * numFamilies * numQualifiers - 2;
754 assertTrue("scannerCount: " + scannerCount + " expectedCount: " + expectedCount,
755 scannerCount == expectedCount);
756
757 scanner = tmpTable.getScanner(scan);
758
759 Put put1 = new Put(rows[0]);
760 put1.add(new KeyValue(rows[0], families[0], qualifiers[0], 1, value));
761 tmpTable.put(put1);
762
763 Put put2 = new Put(rows[1]);
764 put2.add(new KeyValue(rows[1], families[1], qualifiers[1], 2, value));
765 tmpTable.put(put2);
766
767
768 scannerCount = countCellsFromScanner(scanner);
769 expectedCount = numRows * numFamilies * numQualifiers - 2;
770 assertTrue("scannerCount: " + scannerCount + " expectedCount: " + expectedCount,
771 scannerCount == expectedCount);
772
773
774 scanner = tmpTable.getScanner(scan);
775 scannerCount = countCellsFromScanner(scanner);
776 expectedCount = numRows * numFamilies * numQualifiers;
777 assertTrue("scannerCount: " + scannerCount + " expectedCount: " + expectedCount,
778 scannerCount == expectedCount);
779
780 TEST_UTIL.deleteTable(testName);
781 }
782
783
784
785
786
787
788
789
790 private int countCellsFromScanner(ResultScanner scanner) throws Exception {
791 Result result = null;
792 int numCells = 0;
793 while ((result = scanner.next()) != null) {
794 numCells += result.rawCells().length;
795 }
796
797 scanner.close();
798 return numCells;
799 }
800
801
802
803
804
805
806
807
808 @Test
809 public void testPartialResultsWithColumnFilter() throws Exception {
810 testPartialResultsWithColumnFilter(new FirstKeyOnlyFilter());
811 testPartialResultsWithColumnFilter(new ColumnPrefixFilter(Bytes.toBytes("testQualifier5")));
812 testPartialResultsWithColumnFilter(new ColumnRangeFilter(Bytes.toBytes("testQualifer1"), true,
813 Bytes.toBytes("testQualifier7"), true));
814
815 Set<byte[]> qualifiers = new LinkedHashSet<>();
816 qualifiers.add(Bytes.toBytes("testQualifier5"));
817 testPartialResultsWithColumnFilter(new FirstKeyValueMatchingQualifiersFilter(qualifiers));
818 }
819
820 public void testPartialResultsWithColumnFilter(Filter filter) throws Exception {
821 assertTrue(!filter.hasFilterRow());
822
823 Scan partialScan = new Scan();
824 partialScan.setFilter(filter);
825
826 Scan oneshotScan = new Scan();
827 oneshotScan.setFilter(filter);
828 oneshotScan.setMaxResultSize(Long.MAX_VALUE);
829
830 for (int i = 1; i <= NUM_COLS; i++) {
831 partialScan.setMaxResultSize(getResultSizeForNumberOfCells(i));
832 testEquivalenceOfScanResults(TABLE, partialScan, oneshotScan);
833 }
834 }
835
836 private void moveRegion(Table table, int index) throws IOException{
837 List<Pair<HRegionInfo, ServerName>> regions = MetaTableAccessor
838 .getTableRegionsAndLocations(TEST_UTIL.getZooKeeperWatcher(), TEST_UTIL.getConnection(),
839 table.getName());
840 assertEquals(1, regions.size());
841 HRegionInfo regionInfo = regions.get(0).getFirst();
842 ServerName name = TEST_UTIL.getHBaseCluster().getRegionServer(index).getServerName();
843 TEST_UTIL.getHBaseAdmin().move(regionInfo.getEncodedNameAsBytes(),
844 Bytes.toBytes(name.getServerName()));
845 }
846
847 private void assertCell(Cell cell, byte[] row, byte[] cf, byte[] cq) {
848 assertArrayEquals(row,
849 Bytes.copy(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
850 assertArrayEquals(cf,
851 Bytes.copy(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength()));
852 assertArrayEquals(cq,
853 Bytes.copy(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
854 }
855
856 @Test
857 public void testPartialResultWhenRegionMove() throws IOException {
858 Table table=createTestTable(TableName.valueOf("testPartialResultWhenRegionMove"),
859 ROWS, FAMILIES, QUALIFIERS, VALUE);
860
861 moveRegion(table, 1);
862
863 Scan scan = new Scan();
864 scan.setMaxResultSize(1);
865 scan.setAllowPartialResults(true);
866 ResultScanner scanner = table.getScanner(scan);
867 for (int i = 0; i < NUM_FAMILIES * NUM_QUALIFIERS - 1; i++) {
868 scanner.next();
869 }
870 Result result1 = scanner.next();
871 assertEquals(1, result1.rawCells().length);
872 Cell c1 = result1.rawCells()[0];
873 assertCell(c1, ROWS[0], FAMILIES[NUM_FAMILIES - 1], QUALIFIERS[NUM_QUALIFIERS - 1]);
874 assertFalse(result1.isPartial());
875
876 moveRegion(table, 2);
877
878 Result result2 = scanner.next();
879 assertEquals(1, result2.rawCells().length);
880 Cell c2 = result2.rawCells()[0];
881 assertCell(c2, ROWS[1], FAMILIES[0], QUALIFIERS[0]);
882 assertTrue(result2.isPartial());
883
884 moveRegion(table, 3);
885
886 Result result3 = scanner.next();
887 assertEquals(1, result3.rawCells().length);
888 Cell c3 = result3.rawCells()[0];
889 assertCell(c3, ROWS[1], FAMILIES[0], QUALIFIERS[1]);
890 assertTrue(result3.isPartial());
891
892 }
893
894 @Test
895 public void testReversedPartialResultWhenRegionMove() throws IOException {
896 Table table=createTestTable(TableName.valueOf("testReversedPartialResultWhenRegionMove"),
897 ROWS, FAMILIES, QUALIFIERS, VALUE);
898
899 moveRegion(table, 1);
900
901 Scan scan = new Scan();
902 scan.setMaxResultSize(1);
903 scan.setAllowPartialResults(true);
904 scan.setReversed(true);
905 ResultScanner scanner = table.getScanner(scan);
906 for (int i = 0; i < NUM_FAMILIES * NUM_QUALIFIERS-1; i++) {
907 scanner.next();
908 }
909 Result result1 = scanner.next();
910 assertEquals(1, result1.rawCells().length);
911 Cell c1 = result1.rawCells()[0];
912 assertCell(c1, ROWS[NUM_ROWS-1], FAMILIES[NUM_FAMILIES - 1], QUALIFIERS[NUM_QUALIFIERS - 1]);
913 assertFalse(result1.isPartial());
914
915 moveRegion(table, 2);
916
917 Result result2 = scanner.next();
918 assertEquals(1, result2.rawCells().length);
919 Cell c2 = result2.rawCells()[0];
920 assertCell(c2, ROWS[NUM_ROWS-2], FAMILIES[0], QUALIFIERS[0]);
921 assertTrue(result2.isPartial());
922
923 moveRegion(table, 3);
924
925 Result result3 = scanner.next();
926 assertEquals(1, result3.rawCells().length);
927 Cell c3 = result3.rawCells()[0];
928 assertCell(c3, ROWS[NUM_ROWS-2], FAMILIES[0], QUALIFIERS[1]);
929 assertTrue(result3.isPartial());
930
931 }
932
933 @Test
934 public void testCompleteResultWhenRegionMove() throws IOException {
935 Table table=createTestTable(TableName.valueOf("testCompleteResultWhenRegionMove"),
936 ROWS, FAMILIES, QUALIFIERS, VALUE);
937
938 moveRegion(table, 1);
939
940 Scan scan = new Scan();
941 scan.setMaxResultSize(1);
942 scan.setCaching(1);
943 ResultScanner scanner = table.getScanner(scan);
944
945 Result result1 = scanner.next();
946 assertEquals(NUM_FAMILIES * NUM_QUALIFIERS, result1.rawCells().length);
947 Cell c1 = result1.rawCells()[0];
948 assertCell(c1, ROWS[0], FAMILIES[0], QUALIFIERS[0]);
949 assertFalse(result1.isPartial());
950
951 moveRegion(table, 2);
952
953 Result result2 = scanner.next();
954 assertEquals(NUM_FAMILIES * NUM_QUALIFIERS, result2.rawCells().length);
955 Cell c2 = result2.rawCells()[0];
956 assertCell(c2, ROWS[1], FAMILIES[0], QUALIFIERS[0]);
957 assertFalse(result2.isPartial());
958
959 moveRegion(table, 3);
960
961 Result result3 = scanner.next();
962 assertEquals(NUM_FAMILIES * NUM_QUALIFIERS, result3.rawCells().length);
963 Cell c3 = result3.rawCells()[0];
964 assertCell(c3, ROWS[2], FAMILIES[0], QUALIFIERS[0]);
965 assertFalse(result3.isPartial());
966
967 }
968
969 @Test
970 public void testReversedCompleteResultWhenRegionMove() throws IOException {
971 Table table=createTestTable(TableName.valueOf("testReversedCompleteResultWhenRegionMove"),
972 ROWS, FAMILIES, QUALIFIERS, VALUE);
973
974 moveRegion(table, 1);
975
976 Scan scan = new Scan();
977 scan.setMaxResultSize(1);
978 scan.setCaching(1);
979 scan.setReversed(true);
980 ResultScanner scanner = table.getScanner(scan);
981
982 Result result1 = scanner.next();
983 assertEquals(NUM_FAMILIES*NUM_QUALIFIERS, result1.rawCells().length);
984 Cell c1 = result1.rawCells()[0];
985 assertCell(c1, ROWS[NUM_ROWS-1], FAMILIES[0], QUALIFIERS[0]);
986 assertFalse(result1.isPartial());
987
988 moveRegion(table, 2);
989
990 Result result2 = scanner.next();
991 assertEquals(NUM_FAMILIES*NUM_QUALIFIERS, result2.rawCells().length);
992 Cell c2 = result2.rawCells()[0];
993 assertCell(c2, ROWS[NUM_ROWS-2], FAMILIES[0], QUALIFIERS[0]);
994 assertFalse(result2.isPartial());
995
996 moveRegion(table, 3);
997
998 Result result3 = scanner.next();
999 assertEquals(NUM_FAMILIES*NUM_QUALIFIERS, result3.rawCells().length);
1000 Cell c3 = result3.rawCells()[0];
1001 assertCell(c3, ROWS[NUM_ROWS-3], FAMILIES[0], QUALIFIERS[0]);
1002 assertFalse(result3.isPartial());
1003
1004 }
1005
1006 @Test
1007 public void testBatchingResultWhenRegionMove() throws IOException {
1008 Table table =
1009 createTestTable(TableName.valueOf("testBatchingResultWhenRegionMove"), ROWS, FAMILIES,
1010 QUALIFIERS, VALUE);
1011
1012 moveRegion(table, 1);
1013
1014 Scan scan = new Scan();
1015 scan.setCaching(1);
1016 scan.setBatch(1);
1017
1018 ResultScanner scanner = table.getScanner(scan);
1019 for (int i = 0; i < NUM_FAMILIES * NUM_QUALIFIERS - 1; i++) {
1020 scanner.next();
1021 }
1022 Result result1 = scanner.next();
1023 assertEquals(1, result1.rawCells().length);
1024 Cell c1 = result1.rawCells()[0];
1025 assertCell(c1, ROWS[0], FAMILIES[NUM_FAMILIES - 1], QUALIFIERS[NUM_QUALIFIERS - 1]);
1026
1027 moveRegion(table, 2);
1028
1029 Result result2 = scanner.next();
1030 assertEquals(1, result2.rawCells().length);
1031 Cell c2 = result2.rawCells()[0];
1032 assertCell(c2, ROWS[1], FAMILIES[0], QUALIFIERS[0]);
1033
1034 moveRegion(table, 3);
1035
1036 Result result3 = scanner.next();
1037 assertEquals(1, result3.rawCells().length);
1038 Cell c3 = result3.rawCells()[0];
1039 assertCell(c3, ROWS[1], FAMILIES[0], QUALIFIERS[1]);
1040 }
1041
1042
1043 }