1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.io.hfile;
20
21 import java.io.ByteArrayInputStream;
22 import java.io.ByteArrayOutputStream;
23 import java.io.DataInput;
24 import java.io.DataInputStream;
25 import java.io.DataOutputStream;
26 import java.io.IOException;
27 import java.nio.ByteBuffer;
28
29 import org.apache.hadoop.hbase.util.ByteStringer;
30 import org.apache.hadoop.hbase.classification.InterfaceAudience;
31 import org.apache.hadoop.fs.FSDataInputStream;
32 import org.apache.hadoop.hbase.KeyValue;
33 import org.apache.hadoop.hbase.KeyValue.KVComparator;
34 import org.apache.hadoop.hbase.io.compress.Compression;
35 import org.apache.hadoop.hbase.protobuf.generated.HFileProtos;
36 import org.apache.hadoop.hbase.util.Bytes;
37
38
39
40
41
42
43
44
45
46
47
48
49
50 @InterfaceAudience.Private
51 public class FixedFileTrailer {
52
53
54
55 private static final int MAX_COMPARATOR_NAME_LENGTH = 128;
56
57
58
59
60
61 private long fileInfoOffset;
62
63
64
65
66
67
68
69
70 private long loadOnOpenDataOffset;
71
72
73 private int dataIndexCount;
74
75
76 private long uncompressedDataIndexSize;
77
78
79 private int metaIndexCount;
80
81
82 private long totalUncompressedBytes;
83
84
85
86
87
88 private long entryCount;
89
90
91 private Compression.Algorithm compressionCodec = Compression.Algorithm.NONE;
92
93
94
95
96
97 private int numDataIndexLevels;
98
99
100 private long firstDataBlockOffset;
101
102
103
104
105
106 private long lastDataBlockOffset;
107
108
109 private String comparatorClassName = KeyValue.COMPARATOR.getLegacyKeyComparatorName();
110
111
112 private byte[] encryptionKey;
113
114
115 private final int majorVersion;
116
117
118 private final int minorVersion;
119
120 FixedFileTrailer(int majorVersion, int minorVersion) {
121 this.majorVersion = majorVersion;
122 this.minorVersion = minorVersion;
123 HFile.checkFormatVersion(majorVersion);
124 }
125
126 private static int[] computeTrailerSizeByVersion() {
127 int versionToSize[] = new int[HFile.MAX_FORMAT_VERSION + 1];
128
129 versionToSize[2] = 212;
130 for (int version = 3; version <= HFile.MAX_FORMAT_VERSION; version++) {
131
132
133
134 versionToSize[version] = 1024 * 4;
135 }
136 return versionToSize;
137 }
138
139 private static int getMaxTrailerSize() {
140 int maxSize = 0;
141 for (int version = HFile.MIN_FORMAT_VERSION;
142 version <= HFile.MAX_FORMAT_VERSION;
143 ++version)
144 maxSize = Math.max(getTrailerSize(version), maxSize);
145 return maxSize;
146 }
147
148 private static final int TRAILER_SIZE[] = computeTrailerSizeByVersion();
149 private static final int MAX_TRAILER_SIZE = getMaxTrailerSize();
150
151 private static final int NOT_PB_SIZE = BlockType.MAGIC_LENGTH + Bytes.SIZEOF_INT;
152
153 static int getTrailerSize(int version) {
154 return TRAILER_SIZE[version];
155 }
156
157 public int getTrailerSize() {
158 return getTrailerSize(majorVersion);
159 }
160
161
162
163
164
165
166
167
168
169 void serialize(DataOutputStream outputStream) throws IOException {
170 HFile.checkFormatVersion(majorVersion);
171
172 ByteArrayOutputStream baos = new ByteArrayOutputStream();
173 DataOutputStream baosDos = new DataOutputStream(baos);
174
175 BlockType.TRAILER.write(baosDos);
176 serializeAsPB(baosDos);
177
178
179 baosDos.writeInt(materializeVersion(majorVersion, minorVersion));
180
181 baos.writeTo(outputStream);
182 }
183
184
185
186
187
188
189 void serializeAsPB(DataOutputStream output) throws IOException {
190 ByteArrayOutputStream baos = new ByteArrayOutputStream();
191 HFileProtos.FileTrailerProto.Builder builder = HFileProtos.FileTrailerProto.newBuilder()
192 .setFileInfoOffset(fileInfoOffset)
193 .setLoadOnOpenDataOffset(loadOnOpenDataOffset)
194 .setUncompressedDataIndexSize(uncompressedDataIndexSize)
195 .setTotalUncompressedBytes(totalUncompressedBytes)
196 .setDataIndexCount(dataIndexCount)
197 .setMetaIndexCount(metaIndexCount)
198 .setEntryCount(entryCount)
199 .setNumDataIndexLevels(numDataIndexLevels)
200 .setFirstDataBlockOffset(firstDataBlockOffset)
201 .setLastDataBlockOffset(lastDataBlockOffset)
202
203
204 .setComparatorClassName(comparatorClassName)
205 .setCompressionCodec(compressionCodec.ordinal());
206 if (encryptionKey != null) {
207 builder.setEncryptionKey(ByteStringer.wrap(encryptionKey));
208 }
209
210
211 builder.build().writeDelimitedTo(baos);
212 baos.writeTo(output);
213
214
215
216
217 int padding = getTrailerSize() - NOT_PB_SIZE - baos.size();
218 if (padding < 0) {
219 throw new IOException("Pbuf encoding size exceeded fixed trailer size limit");
220 }
221 for (int i = 0; i < padding; i++) {
222 output.write(0);
223 }
224 }
225
226
227
228
229
230
231
232
233
234 void deserialize(DataInputStream inputStream) throws IOException {
235 HFile.checkFormatVersion(majorVersion);
236
237 BlockType.TRAILER.readAndCheck(inputStream);
238
239 if (majorVersion > 2
240 || (majorVersion == 2 && minorVersion >= HFileReaderV2.PBUF_TRAILER_MINOR_VERSION)) {
241 deserializeFromPB(inputStream);
242 } else {
243 deserializeFromWritable(inputStream);
244 }
245
246
247 int version = inputStream.readInt();
248 expectMajorVersion(extractMajorVersion(version));
249 expectMinorVersion(extractMinorVersion(version));
250 }
251
252
253
254
255
256
257 void deserializeFromPB(DataInputStream inputStream) throws IOException {
258
259 int start = inputStream.available();
260 HFileProtos.FileTrailerProto trailerProto =
261 HFileProtos.FileTrailerProto.PARSER.parseDelimitedFrom(inputStream);
262 int size = start - inputStream.available();
263 inputStream.skip(getTrailerSize() - NOT_PB_SIZE - size);
264
265
266 if (trailerProto.hasFileInfoOffset()) {
267 fileInfoOffset = trailerProto.getFileInfoOffset();
268 }
269 if (trailerProto.hasLoadOnOpenDataOffset()) {
270 loadOnOpenDataOffset = trailerProto.getLoadOnOpenDataOffset();
271 }
272 if (trailerProto.hasUncompressedDataIndexSize()) {
273 uncompressedDataIndexSize = trailerProto.getUncompressedDataIndexSize();
274 }
275 if (trailerProto.hasTotalUncompressedBytes()) {
276 totalUncompressedBytes = trailerProto.getTotalUncompressedBytes();
277 }
278 if (trailerProto.hasDataIndexCount()) {
279 dataIndexCount = trailerProto.getDataIndexCount();
280 }
281 if (trailerProto.hasMetaIndexCount()) {
282 metaIndexCount = trailerProto.getMetaIndexCount();
283 }
284 if (trailerProto.hasEntryCount()) {
285 entryCount = trailerProto.getEntryCount();
286 }
287 if (trailerProto.hasNumDataIndexLevels()) {
288 numDataIndexLevels = trailerProto.getNumDataIndexLevels();
289 }
290 if (trailerProto.hasFirstDataBlockOffset()) {
291 firstDataBlockOffset = trailerProto.getFirstDataBlockOffset();
292 }
293 if (trailerProto.hasLastDataBlockOffset()) {
294 lastDataBlockOffset = trailerProto.getLastDataBlockOffset();
295 }
296 if (trailerProto.hasComparatorClassName()) {
297
298
299 setComparatorClass(getComparatorClass(trailerProto.getComparatorClassName()));
300 }
301 if (trailerProto.hasCompressionCodec()) {
302 compressionCodec = Compression.Algorithm.values()[trailerProto.getCompressionCodec()];
303 } else {
304 compressionCodec = Compression.Algorithm.NONE;
305 }
306 if (trailerProto.hasEncryptionKey()) {
307 encryptionKey = trailerProto.getEncryptionKey().toByteArray();
308 }
309 }
310
311
312
313
314
315
316 void deserializeFromWritable(DataInput input) throws IOException {
317 fileInfoOffset = input.readLong();
318 loadOnOpenDataOffset = input.readLong();
319 dataIndexCount = input.readInt();
320 uncompressedDataIndexSize = input.readLong();
321 metaIndexCount = input.readInt();
322
323 totalUncompressedBytes = input.readLong();
324 entryCount = input.readLong();
325 compressionCodec = Compression.Algorithm.values()[input.readInt()];
326 numDataIndexLevels = input.readInt();
327 firstDataBlockOffset = input.readLong();
328 lastDataBlockOffset = input.readLong();
329
330
331 setComparatorClass(getComparatorClass(Bytes.readStringFixedSize(input,
332 MAX_COMPARATOR_NAME_LENGTH)));
333 }
334
335 private void append(StringBuilder sb, String s) {
336 if (sb.length() > 0)
337 sb.append(", ");
338 sb.append(s);
339 }
340
341 @Override
342 public String toString() {
343 StringBuilder sb = new StringBuilder();
344 append(sb, "fileinfoOffset=" + fileInfoOffset);
345 append(sb, "loadOnOpenDataOffset=" + loadOnOpenDataOffset);
346 append(sb, "dataIndexCount=" + dataIndexCount);
347 append(sb, "metaIndexCount=" + metaIndexCount);
348 append(sb, "totalUncomressedBytes=" + totalUncompressedBytes);
349 append(sb, "entryCount=" + entryCount);
350 append(sb, "compressionCodec=" + compressionCodec);
351 append(sb, "uncompressedDataIndexSize=" + uncompressedDataIndexSize);
352 append(sb, "numDataIndexLevels=" + numDataIndexLevels);
353 append(sb, "firstDataBlockOffset=" + firstDataBlockOffset);
354 append(sb, "lastDataBlockOffset=" + lastDataBlockOffset);
355 append(sb, "comparatorClassName=" + comparatorClassName);
356 if (majorVersion >= 3) {
357 append(sb, "encryptionKey=" + (encryptionKey != null ? "PRESENT" : "NONE"));
358 }
359 append(sb, "majorVersion=" + majorVersion);
360 append(sb, "minorVersion=" + minorVersion);
361
362 return sb.toString();
363 }
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378 public static FixedFileTrailer readFromStream(FSDataInputStream istream,
379 long fileSize) throws IOException {
380 int bufferSize = MAX_TRAILER_SIZE;
381 long seekPoint = fileSize - bufferSize;
382 if (seekPoint < 0) {
383
384 seekPoint = 0;
385 bufferSize = (int) fileSize;
386 }
387
388 istream.seek(seekPoint);
389 ByteBuffer buf = ByteBuffer.allocate(bufferSize);
390 istream.readFully(buf.array(), buf.arrayOffset(),
391 buf.arrayOffset() + buf.limit());
392
393
394 buf.position(buf.limit() - Bytes.SIZEOF_INT);
395 int version = buf.getInt();
396
397
398 int majorVersion = extractMajorVersion(version);
399 int minorVersion = extractMinorVersion(version);
400
401 HFile.checkFormatVersion(majorVersion);
402
403 int trailerSize = getTrailerSize(majorVersion);
404
405 FixedFileTrailer fft = new FixedFileTrailer(majorVersion, minorVersion);
406 fft.deserialize(new DataInputStream(new ByteArrayInputStream(buf.array(),
407 buf.arrayOffset() + bufferSize - trailerSize, trailerSize)));
408 return fft;
409 }
410
411 public void expectMajorVersion(int expected) {
412 if (majorVersion != expected) {
413 throw new IllegalArgumentException("Invalid HFile major version: "
414 + majorVersion
415 + " (expected: " + expected + ")");
416 }
417 }
418
419 public void expectMinorVersion(int expected) {
420 if (minorVersion != expected) {
421 throw new IllegalArgumentException("Invalid HFile minor version: "
422 + minorVersion + " (expected: " + expected + ")");
423 }
424 }
425
426 public void expectAtLeastMajorVersion(int lowerBound) {
427 if (majorVersion < lowerBound) {
428 throw new IllegalArgumentException("Invalid HFile major version: "
429 + majorVersion
430 + " (expected: " + lowerBound + " or higher).");
431 }
432 }
433
434 public long getFileInfoOffset() {
435 return fileInfoOffset;
436 }
437
438 public void setFileInfoOffset(long fileInfoOffset) {
439 this.fileInfoOffset = fileInfoOffset;
440 }
441
442 public long getLoadOnOpenDataOffset() {
443 return loadOnOpenDataOffset;
444 }
445
446 public void setLoadOnOpenOffset(long loadOnOpenDataOffset) {
447 this.loadOnOpenDataOffset = loadOnOpenDataOffset;
448 }
449
450 public int getDataIndexCount() {
451 return dataIndexCount;
452 }
453
454 public void setDataIndexCount(int dataIndexCount) {
455 this.dataIndexCount = dataIndexCount;
456 }
457
458 public int getMetaIndexCount() {
459 return metaIndexCount;
460 }
461
462 public void setMetaIndexCount(int metaIndexCount) {
463 this.metaIndexCount = metaIndexCount;
464 }
465
466 public long getTotalUncompressedBytes() {
467 return totalUncompressedBytes;
468 }
469
470 public void setTotalUncompressedBytes(long totalUncompressedBytes) {
471 this.totalUncompressedBytes = totalUncompressedBytes;
472 }
473
474 public long getEntryCount() {
475 return entryCount;
476 }
477
478 public void setEntryCount(long newEntryCount) {
479 entryCount = newEntryCount;
480 }
481
482 public Compression.Algorithm getCompressionCodec() {
483 return compressionCodec;
484 }
485
486 public void setCompressionCodec(Compression.Algorithm compressionCodec) {
487 this.compressionCodec = compressionCodec;
488 }
489
490 public int getNumDataIndexLevels() {
491 expectAtLeastMajorVersion(2);
492 return numDataIndexLevels;
493 }
494
495 public void setNumDataIndexLevels(int numDataIndexLevels) {
496 expectAtLeastMajorVersion(2);
497 this.numDataIndexLevels = numDataIndexLevels;
498 }
499
500 public long getLastDataBlockOffset() {
501 expectAtLeastMajorVersion(2);
502 return lastDataBlockOffset;
503 }
504
505 public void setLastDataBlockOffset(long lastDataBlockOffset) {
506 expectAtLeastMajorVersion(2);
507 this.lastDataBlockOffset = lastDataBlockOffset;
508 }
509
510 public long getFirstDataBlockOffset() {
511 expectAtLeastMajorVersion(2);
512 return firstDataBlockOffset;
513 }
514
515 public void setFirstDataBlockOffset(long firstDataBlockOffset) {
516 expectAtLeastMajorVersion(2);
517 this.firstDataBlockOffset = firstDataBlockOffset;
518 }
519
520 public String getComparatorClassName() {
521 return comparatorClassName;
522 }
523
524
525
526
527 public int getMajorVersion() {
528 return majorVersion;
529 }
530
531
532
533
534 public int getMinorVersion() {
535 return minorVersion;
536 }
537
538 public void setComparatorClass(Class<? extends KVComparator> klass) {
539
540 try {
541 KVComparator comp = klass.newInstance();
542
543
544 if (KeyValue.COMPARATOR.getClass().equals(klass)) {
545 comparatorClassName = KeyValue.COMPARATOR.getLegacyKeyComparatorName();
546 } else if (KeyValue.META_COMPARATOR.getClass().equals(klass)) {
547 comparatorClassName = KeyValue.META_COMPARATOR.getLegacyKeyComparatorName();
548 } else if (KeyValue.RAW_COMPARATOR.getClass().equals(klass)) {
549 comparatorClassName = KeyValue.RAW_COMPARATOR.getLegacyKeyComparatorName();
550 } else {
551
552 comparatorClassName = klass.getName();
553 }
554
555 } catch (Exception e) {
556 throw new RuntimeException("Comparator class " + klass.getName() +
557 " is not instantiable", e);
558 }
559
560 }
561
562 @SuppressWarnings("unchecked")
563 private static Class<? extends KVComparator> getComparatorClass(
564 String comparatorClassName) throws IOException {
565 try {
566
567 if (comparatorClassName.equals(KeyValue.COMPARATOR.getLegacyKeyComparatorName())) {
568 comparatorClassName = KeyValue.COMPARATOR.getClass().getName();
569 } else if (comparatorClassName.equals(KeyValue.META_COMPARATOR.getLegacyKeyComparatorName())) {
570 comparatorClassName = KeyValue.META_COMPARATOR.getClass().getName();
571 } else if (comparatorClassName.equals(KeyValue.RAW_COMPARATOR.getLegacyKeyComparatorName())) {
572 comparatorClassName = KeyValue.RAW_COMPARATOR.getClass().getName();
573 }
574
575
576
577 return (Class<? extends KVComparator>)
578 Class.forName(comparatorClassName);
579 } catch (ClassNotFoundException ex) {
580 throw new IOException(ex);
581 }
582 }
583
584 public static KVComparator createComparator(
585 String comparatorClassName) throws IOException {
586 try {
587 return getComparatorClass(comparatorClassName).newInstance();
588 } catch (InstantiationException e) {
589 throw new IOException("Comparator class " + comparatorClassName +
590 " is not instantiable", e);
591 } catch (IllegalAccessException e) {
592 throw new IOException("Comparator class " + comparatorClassName +
593 " is not instantiable", e);
594 }
595 }
596
597 KVComparator createComparator() throws IOException {
598 expectAtLeastMajorVersion(2);
599 return createComparator(comparatorClassName);
600 }
601
602 public long getUncompressedDataIndexSize() {
603 return uncompressedDataIndexSize;
604 }
605
606 public void setUncompressedDataIndexSize(
607 long uncompressedDataIndexSize) {
608 expectAtLeastMajorVersion(2);
609 this.uncompressedDataIndexSize = uncompressedDataIndexSize;
610 }
611
612 public byte[] getEncryptionKey() {
613 expectAtLeastMajorVersion(3);
614 return encryptionKey;
615 }
616
617 public void setEncryptionKey(byte[] keyBytes) {
618 this.encryptionKey = keyBytes;
619 }
620
621
622
623
624
625 private static int extractMajorVersion(int serializedVersion) {
626 return (serializedVersion & 0x00ffffff);
627 }
628
629
630
631
632
633 private static int extractMinorVersion(int serializedVersion) {
634 return (serializedVersion >>> 24);
635 }
636
637
638
639
640
641 static int materializeVersion(int majorVersion, int minorVersion) {
642 return ((majorVersion & 0x00ffffff) | (minorVersion << 24));
643 }
644 }