View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertNull;
22  
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.Arrays;
26  import java.util.Collections;
27  import java.util.List;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileStatus;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.hbase.HBaseTestingUtility;
36  import org.apache.hadoop.hbase.regionserver.HRegion;
37  import org.apache.hadoop.hbase.regionserver.Store;
38  
39  /**
40   * Test helper for testing archiving of HFiles
41   */
42  public class HFileArchiveTestingUtil {
43  
44    private static final Log LOG = LogFactory.getLog(HFileArchiveTestingUtil.class);
45  
46    private HFileArchiveTestingUtil() {
47      // NOOP private ctor since this is just a utility class
48    }
49  
50    public static boolean compareArchiveToOriginal(FileStatus[] previous, FileStatus[] archived,
51        FileSystem fs, boolean hasTimedBackup) {
52  
53      List<List<String>> lists = getFileLists(previous, archived);
54      List<String> original = lists.get(0);
55      Collections.sort(original);
56  
57      List<String> currentFiles = lists.get(1);
58      Collections.sort(currentFiles);
59  
60      List<String> backedup = lists.get(2);
61      Collections.sort(backedup);
62  
63      // check the backed up files versus the current (should match up, less the
64      // backup time in the name)
65      if (!hasTimedBackup == (backedup.size() > 0)) {
66        LOG.debug("backedup files doesn't match expected.");
67        return false;
68      }
69      String msg = null;
70      if (hasTimedBackup) {
71        msg = assertArchiveEquality(original, backedup);
72        if (msg != null) {
73          LOG.debug(msg);
74          return false;
75        }
76      }
77      msg = assertArchiveEquality(original, currentFiles);
78      if (msg != null) {
79        LOG.debug(msg);
80        return false;
81      }
82      return true;
83    }
84  
85    /**
86     * Compare the archived files to the files in the original directory
87     * @param expected original files that should have been archived
88     * @param actual files that were archived
89     * @param fs filessystem on which the archiving took place
90     * @throws IOException
91     */
92    public static void assertArchiveEqualToOriginal(FileStatus[] expected, FileStatus[] actual,
93        FileSystem fs) throws IOException {
94      assertArchiveEqualToOriginal(expected, actual, fs, false);
95    }
96  
97    /**
98     * Compare the archived files to the files in the original directory
99     * @param expected original files that should have been archived
100    * @param actual files that were archived
101    * @param fs {@link FileSystem} on which the archiving took place
102    * @param hasTimedBackup <tt>true</tt> if we expect to find an archive backup directory with a
103    *          copy of the files in the archive directory (and the original files).
104    * @throws IOException
105    */
106   public static void assertArchiveEqualToOriginal(FileStatus[] expected, FileStatus[] actual,
107       FileSystem fs, boolean hasTimedBackup) throws IOException {
108 
109     List<List<String>> lists = getFileLists(expected, actual);
110     List<String> original = lists.get(0);
111     Collections.sort(original);
112 
113     List<String> currentFiles = lists.get(1);
114     Collections.sort(currentFiles);
115 
116     List<String> backedup = lists.get(2);
117     Collections.sort(backedup);
118 
119     // check the backed up files versus the current (should match up, less the
120     // backup time in the name)
121     assertEquals("Didn't expect any backup files, but got: " + backedup, hasTimedBackup,
122       backedup.size() > 0);
123     String msg = null;
124     if (hasTimedBackup) {
125       assertArchiveEquality(original, backedup);
126       assertNull(msg, msg);
127     }
128 
129     // do the rest of the comparison
130     msg = assertArchiveEquality(original, currentFiles);
131     assertNull(msg, msg);
132   }
133 
134   private static String assertArchiveEquality(List<String> expected, List<String> archived) {
135     String compare = compareFileLists(expected, archived);
136     if (!(expected.size() == archived.size())) return "Not the same number of current files\n"
137         + compare;
138     if (!expected.equals(archived)) return "Different backup files, but same amount\n" + compare;
139     return null;
140   }
141 
142   /**
143    * @return <expected, gotten, backup>, where each is sorted
144    */
145   private static List<List<String>> getFileLists(FileStatus[] previous, FileStatus[] archived) {
146     List<List<String>> files = new ArrayList<List<String>>();
147 
148     // copy over the original files
149     List<String> originalFileNames = convertToString(previous);
150     files.add(originalFileNames);
151 
152     List<String> currentFiles = new ArrayList<String>(previous.length);
153     List<FileStatus> backedupFiles = new ArrayList<FileStatus>(previous.length);
154     for (FileStatus f : archived) {
155       String name = f.getPath().getName();
156       // if the file has been backed up
157       if (name.contains(".")) {
158         Path parent = f.getPath().getParent();
159         String shortName = name.split("[.]")[0];
160         Path modPath = new Path(parent, shortName);
161         FileStatus file = new FileStatus(f.getLen(), f.isDirectory(), f.getReplication(),
162             f.getBlockSize(), f.getModificationTime(), modPath);
163         backedupFiles.add(file);
164       } else {
165         // otherwise, add it to the list to compare to the original store files
166         currentFiles.add(name);
167       }
168     }
169 
170     files.add(currentFiles);
171     files.add(convertToString(backedupFiles));
172     return files;
173   }
174 
175   private static List<String> convertToString(FileStatus[] files) {
176     return convertToString(Arrays.asList(files));
177   }
178 
179   private static List<String> convertToString(List<FileStatus> files) {
180     List<String> originalFileNames = new ArrayList<String>(files.size());
181     for (FileStatus f : files) {
182       originalFileNames.add(f.getPath().getName());
183     }
184     return originalFileNames;
185   }
186 
187   /* Get a pretty representation of the differences */
188   private static String compareFileLists(List<String> expected, List<String> gotten) {
189     StringBuilder sb = new StringBuilder("Expected (" + expected.size() + "): \t\t Gotten ("
190         + gotten.size() + "):\n");
191     List<String> notFound = new ArrayList<String>();
192     for (String s : expected) {
193       if (gotten.contains(s)) sb.append(s + "\t\t" + s + "\n");
194       else notFound.add(s);
195     }
196     sb.append("Not Found:\n");
197     for (String s : notFound) {
198       sb.append(s + "\n");
199     }
200     sb.append("\nExtra:\n");
201     for (String s : gotten) {
202       if (!expected.contains(s)) sb.append(s + "\n");
203     }
204     return sb.toString();
205   }
206 
207   /**
208    * Helper method to get the archive directory for the specified region
209    * @param conf {@link Configuration} to check for the name of the archive directory
210    * @param region region that is being archived
211    * @return {@link Path} to the archive directory for the given region
212    */
213   public static Path getRegionArchiveDir(Configuration conf, HRegion region) throws IOException {
214     return HFileArchiveUtil.getRegionArchiveDir(
215         FSUtils.getRootDir(conf),
216         region.getTableDesc().getTableName(),
217         region.getRegionInfo().getEncodedName());
218   }
219 
220   /**
221    * Helper method to get the store archive directory for the specified region
222    * @param conf {@link Configuration} to check for the name of the archive directory
223    * @param region region that is being archived
224    * @param store store that is archiving files
225    * @return {@link Path} to the store archive directory for the given region
226    */
227   public static Path getStoreArchivePath(Configuration conf, HRegion region, Store store)
228       throws IOException {
229     return HFileArchiveUtil.getStoreArchivePath(conf, region.getRegionInfo(),
230         region.getRegionFileSystem().getTableDir(), store.getFamily().getName());
231   }
232 
233   public static Path getStoreArchivePath(HBaseTestingUtility util, String tableName,
234       byte[] storeName) throws IOException {
235     byte[] table = Bytes.toBytes(tableName);
236     // get the RS and region serving our table
237     List<HRegion> servingRegions = util.getHBaseCluster().getRegions(table);
238     HRegion region = servingRegions.get(0);
239 
240     // check that we actually have some store files that were archived
241     Store store = region.getStore(storeName);
242     return HFileArchiveTestingUtil.getStoreArchivePath(util.getConfiguration(), region, store);
243   }
244 }