1 /** 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 package org.apache.hadoop.hbase.snapshot; 19 20 import java.io.IOException; 21 import java.util.Collections; 22 23 import org.apache.commons.logging.Log; 24 import org.apache.commons.logging.LogFactory; 25 import org.apache.hadoop.conf.Configuration; 26 import org.apache.hadoop.fs.FSDataInputStream; 27 import org.apache.hadoop.fs.FSDataOutputStream; 28 import org.apache.hadoop.fs.FileSystem; 29 import org.apache.hadoop.fs.Path; 30 import org.apache.hadoop.fs.permission.FsPermission; 31 import org.apache.hadoop.hbase.HConstants; 32 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; 33 import org.apache.hadoop.hbase.security.User; 34 import org.apache.hadoop.hbase.snapshot.SnapshotManifestV2; 35 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 36 import org.apache.hadoop.hbase.util.FSUtils; 37 38 /** 39 * Utility class to help manage {@link SnapshotDescription SnapshotDesriptions}. 40 * <p> 41 * Snapshots are laid out on disk like this: 42 * 43 * <pre> 44 * /hbase/.snapshots 45 * /.tmp <---- working directory 46 * /[snapshot name] <----- completed snapshot 47 * </pre> 48 * 49 * A completed snapshot named 'completed' then looks like (multiple regions, servers, files, etc. 50 * signified by '...' on the same directory depth). 51 * 52 * <pre> 53 * /hbase/.snapshots/completed 54 * .snapshotinfo <--- Description of the snapshot 55 * .tableinfo <--- Copy of the tableinfo 56 * /.logs 57 * /[server_name] 58 * /... [log files] 59 * ... 60 * /[region name] <---- All the region's information 61 * .regioninfo <---- Copy of the HRegionInfo 62 * /[column family name] 63 * /[hfile name] <--- name of the hfile in the real region 64 * ... 65 * ... 66 * ... 67 * </pre> 68 * 69 * Utility methods in this class are useful for getting the correct locations for different parts of 70 * the snapshot, as well as moving completed snapshots into place (see 71 * {@link #completeSnapshot}, and writing the 72 * {@link SnapshotDescription} to the working snapshot directory. 73 */ 74 public class SnapshotDescriptionUtils { 75 76 /** 77 * Filter that only accepts completed snapshot directories 78 */ 79 public static class CompletedSnaphotDirectoriesFilter extends FSUtils.BlackListDirFilter { 80 81 /** 82 * @param fs 83 */ 84 public CompletedSnaphotDirectoriesFilter(FileSystem fs) { 85 super(fs, Collections.singletonList(SNAPSHOT_TMP_DIR_NAME)); 86 } 87 } 88 89 private static final Log LOG = LogFactory.getLog(SnapshotDescriptionUtils.class); 90 /** 91 * Version of the fs layout for a snapshot. Future snapshots may have different file layouts, 92 * which we may need to read in differently. 93 */ 94 public static final int SNAPSHOT_LAYOUT_VERSION = SnapshotManifestV2.DESCRIPTOR_VERSION; 95 96 // snapshot directory constants 97 /** 98 * The file contains the snapshot basic information and it is under the directory of a snapshot. 99 */ 100 public static final String SNAPSHOTINFO_FILE = ".snapshotinfo"; 101 102 /** Temporary directory under the snapshot directory to store in-progress snapshots */ 103 public static final String SNAPSHOT_TMP_DIR_NAME = ".tmp"; 104 // snapshot operation values 105 /** Default value if no start time is specified */ 106 public static final long NO_SNAPSHOT_START_TIME_SPECIFIED = 0; 107 108 109 public static final String MASTER_SNAPSHOT_TIMEOUT_MILLIS = "hbase.snapshot.master.timeout.millis"; 110 111 /** By default, wait 300 seconds for a snapshot to complete */ 112 public static final long DEFAULT_MAX_WAIT_TIME = 60000 * 5 ; 113 114 115 /** 116 * By default, check to see if the snapshot is complete (ms) 117 * @deprecated Use {@link #DEFAULT_MAX_WAIT_TIME} instead. 118 * */ 119 @Deprecated 120 public static final int SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 60000 * 5; 121 122 /** 123 * Conf key for # of ms elapsed before injecting a snapshot timeout error when waiting for 124 * completion. 125 * @deprecated Use {@link #MASTER_SNAPSHOT_TIMEOUT_MILLIS} instead. 126 */ 127 @Deprecated 128 public static final String SNAPSHOT_TIMEOUT_MILLIS_KEY = "hbase.snapshot.master.timeoutMillis"; 129 130 private SnapshotDescriptionUtils() { 131 // private constructor for utility class 132 } 133 134 /** 135 * @param conf {@link Configuration} from which to check for the timeout 136 * @param type type of snapshot being taken 137 * @param defaultMaxWaitTime Default amount of time to wait, if none is in the configuration 138 * @return the max amount of time the master should wait for a snapshot to complete 139 */ 140 public static long getMaxMasterTimeout(Configuration conf, SnapshotDescription.Type type, 141 long defaultMaxWaitTime) { 142 String confKey; 143 switch (type) { 144 case DISABLED: 145 default: 146 confKey = MASTER_SNAPSHOT_TIMEOUT_MILLIS; 147 } 148 return Math.max(conf.getLong(confKey, defaultMaxWaitTime), 149 conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, defaultMaxWaitTime)); 150 } 151 152 /** 153 * Get the snapshot root directory. All the snapshots are kept under this directory, i.e. 154 * ${hbase.rootdir}/.snapshot 155 * @param rootDir hbase root directory 156 * @return the base directory in which all snapshots are kept 157 */ 158 public static Path getSnapshotRootDir(final Path rootDir) { 159 return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME); 160 } 161 162 /** 163 * Get the directory for a specified snapshot. This directory is a sub-directory of snapshot root 164 * directory and all the data files for a snapshot are kept under this directory. 165 * @param snapshot snapshot being taken 166 * @param rootDir hbase root directory 167 * @return the final directory for the completed snapshot 168 */ 169 public static Path getCompletedSnapshotDir(final SnapshotDescription snapshot, final Path rootDir) { 170 return getCompletedSnapshotDir(snapshot.getName(), rootDir); 171 } 172 173 /** 174 * Get the directory for a completed snapshot. This directory is a sub-directory of snapshot root 175 * directory and all the data files for a snapshot are kept under this directory. 176 * @param snapshotName name of the snapshot being taken 177 * @param rootDir hbase root directory 178 * @return the final directory for the completed snapshot 179 */ 180 public static Path getCompletedSnapshotDir(final String snapshotName, final Path rootDir) { 181 return getCompletedSnapshotDir(getSnapshotsDir(rootDir), snapshotName); 182 } 183 184 /** 185 * Get the general working directory for snapshots - where they are built, where they are 186 * temporarily copied on export, etc. 187 * @param rootDir root directory of the HBase installation 188 * @return Path to the snapshot tmp directory, relative to the passed root directory 189 */ 190 public static Path getWorkingSnapshotDir(final Path rootDir) { 191 return new Path(getSnapshotsDir(rootDir), SNAPSHOT_TMP_DIR_NAME); 192 } 193 194 /** 195 * Get the directory to build a snapshot, before it is finalized 196 * @param snapshot snapshot that will be built 197 * @param rootDir root directory of the hbase installation 198 * @return {@link Path} where one can build a snapshot 199 */ 200 public static Path getWorkingSnapshotDir(SnapshotDescription snapshot, final Path rootDir) { 201 return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshot.getName()); 202 } 203 204 /** 205 * Get the directory to build a snapshot, before it is finalized 206 * @param snapshotName name of the snapshot 207 * @param rootDir root directory of the hbase installation 208 * @return {@link Path} where one can build a snapshot 209 */ 210 public static Path getWorkingSnapshotDir(String snapshotName, final Path rootDir) { 211 return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshotName); 212 } 213 214 /** 215 * Get the directory to store the snapshot instance 216 * @param snapshotsDir hbase-global directory for storing all snapshots 217 * @param snapshotName name of the snapshot to take 218 * @return the final directory for the completed snapshot 219 */ 220 private static final Path getCompletedSnapshotDir(final Path snapshotsDir, String snapshotName) { 221 return new Path(snapshotsDir, snapshotName); 222 } 223 224 /** 225 * @param rootDir hbase root directory 226 * @return the directory for all completed snapshots; 227 */ 228 public static final Path getSnapshotsDir(Path rootDir) { 229 return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME); 230 } 231 232 /** 233 * Convert the passed snapshot description into a 'full' snapshot description based on default 234 * parameters, if none have been supplied. This resolves any 'optional' parameters that aren't 235 * supplied to their default values. 236 * @param snapshot general snapshot descriptor 237 * @param conf Configuration to read configured snapshot defaults if snapshot is not complete 238 * @return a valid snapshot description 239 * @throws IllegalArgumentException if the {@link SnapshotDescription} is not a complete 240 * {@link SnapshotDescription}. 241 */ 242 public static SnapshotDescription validate(SnapshotDescription snapshot, Configuration conf) 243 throws IllegalArgumentException { 244 if (!snapshot.hasTable()) { 245 throw new IllegalArgumentException( 246 "Descriptor doesn't apply to a table, so we can't build it."); 247 } 248 249 // set the creation time, if one hasn't been set 250 long time = snapshot.getCreationTime(); 251 if (time == SnapshotDescriptionUtils.NO_SNAPSHOT_START_TIME_SPECIFIED) { 252 time = EnvironmentEdgeManager.currentTime(); 253 LOG.debug("Creation time not specified, setting to:" + time + " (current time:" 254 + EnvironmentEdgeManager.currentTime() + ")."); 255 SnapshotDescription.Builder builder = snapshot.toBuilder(); 256 builder.setCreationTime(time); 257 snapshot = builder.build(); 258 } 259 return snapshot; 260 } 261 262 /** 263 * Write the snapshot description into the working directory of a snapshot 264 * @param snapshot description of the snapshot being taken 265 * @param workingDir working directory of the snapshot 266 * @param fs {@link FileSystem} on which the snapshot should be taken 267 * @throws IOException if we can't reach the filesystem and the file cannot be cleaned up on 268 * failure 269 */ 270 public static void writeSnapshotInfo(SnapshotDescription snapshot, Path workingDir, FileSystem fs) 271 throws IOException { 272 FsPermission perms = FSUtils.getFilePermissions(fs, fs.getConf(), 273 HConstants.DATA_FILE_UMASK_KEY); 274 Path snapshotInfo = new Path(workingDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE); 275 try { 276 FSDataOutputStream out = FSUtils.create(fs, snapshotInfo, perms, true); 277 try { 278 snapshot.writeTo(out); 279 } finally { 280 out.close(); 281 } 282 } catch (IOException e) { 283 // if we get an exception, try to remove the snapshot info 284 if (!fs.delete(snapshotInfo, false)) { 285 String msg = "Couldn't delete snapshot info file: " + snapshotInfo; 286 LOG.error(msg); 287 throw new IOException(msg); 288 } 289 } 290 } 291 292 /** 293 * Read in the {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription} stored for the snapshot in the passed directory 294 * @param fs filesystem where the snapshot was taken 295 * @param snapshotDir directory where the snapshot was stored 296 * @return the stored snapshot description 297 * @throws CorruptedSnapshotException if the 298 * snapshot cannot be read 299 */ 300 public static SnapshotDescription readSnapshotInfo(FileSystem fs, Path snapshotDir) 301 throws CorruptedSnapshotException { 302 Path snapshotInfo = new Path(snapshotDir, SNAPSHOTINFO_FILE); 303 try { 304 FSDataInputStream in = null; 305 try { 306 in = fs.open(snapshotInfo); 307 SnapshotDescription desc = SnapshotDescription.parseFrom(in); 308 return desc; 309 } finally { 310 if (in != null) in.close(); 311 } 312 } catch (IOException e) { 313 throw new CorruptedSnapshotException("Couldn't read snapshot info from:" + snapshotInfo, e); 314 } 315 } 316 317 /** 318 * Move the finished snapshot to its final, publicly visible directory - this marks the snapshot 319 * as 'complete'. 320 * @param snapshot description of the snapshot being tabken 321 * @param rootdir root directory of the hbase installation 322 * @param workingDir directory where the in progress snapshot was built 323 * @param fs {@link FileSystem} where the snapshot was built 324 * @throws org.apache.hadoop.hbase.snapshot.SnapshotCreationException if the 325 * snapshot could not be moved 326 * @throws IOException the filesystem could not be reached 327 */ 328 public static void completeSnapshot(SnapshotDescription snapshot, Path rootdir, Path workingDir, 329 FileSystem fs) throws SnapshotCreationException, IOException { 330 Path finishedDir = getCompletedSnapshotDir(snapshot, rootdir); 331 LOG.debug("Snapshot is done, just moving the snapshot from " + workingDir + " to " 332 + finishedDir); 333 if (!fs.rename(workingDir, finishedDir)) { 334 throw new SnapshotCreationException("Failed to move working directory(" + workingDir 335 + ") to completed directory(" + finishedDir + ").", snapshot); 336 } 337 } 338 339 /** 340 * Check if the user is this table snapshot's owner 341 * @param snapshot the table snapshot description 342 * @param user the user 343 * @return true if the user is the owner of the snapshot, 344 * false otherwise or the snapshot owner field is not present. 345 */ 346 public static boolean isSnapshotOwner(final SnapshotDescription snapshot, final User user) { 347 if (user == null) return false; 348 if (!snapshot.hasOwner()) return false; 349 return snapshot.getOwner().equals(user.getShortName()); 350 } 351 }