1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master.procedure;
20
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.io.OutputStream;
24 import java.security.PrivilegedExceptionAction;
25 import java.util.HashMap;
26 import java.util.HashSet;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.concurrent.atomic.AtomicBoolean;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.hbase.HRegionInfo;
34 import org.apache.hadoop.hbase.MetaTableAccessor;
35 import org.apache.hadoop.hbase.ServerName;
36 import org.apache.hadoop.hbase.TableName;
37 import org.apache.hadoop.hbase.TableNotDisabledException;
38 import org.apache.hadoop.hbase.TableNotFoundException;
39 import org.apache.hadoop.hbase.TableStateManager;
40 import org.apache.hadoop.hbase.classification.InterfaceAudience;
41 import org.apache.hadoop.hbase.exceptions.HBaseException;
42 import org.apache.hadoop.hbase.executor.EventType;
43 import org.apache.hadoop.hbase.master.AssignmentManager;
44 import org.apache.hadoop.hbase.master.BulkAssigner;
45 import org.apache.hadoop.hbase.master.GeneralBulkAssigner;
46 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
47 import org.apache.hadoop.hbase.master.MasterServices;
48 import org.apache.hadoop.hbase.master.RegionStates;
49 import org.apache.hadoop.hbase.master.ServerManager;
50 import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
51 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
52 import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos;
53 import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.EnableTableState;
54 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
55 import org.apache.hadoop.hbase.util.Pair;
56 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
57 import org.apache.hadoop.security.UserGroupInformation;
58
59 @InterfaceAudience.Private
60 public class EnableTableProcedure
61 extends StateMachineProcedure<MasterProcedureEnv, EnableTableState>
62 implements TableProcedureInterface {
63 private static final Log LOG = LogFactory.getLog(EnableTableProcedure.class);
64
65 private final AtomicBoolean aborted = new AtomicBoolean(false);
66
67
68 private final ProcedurePrepareLatch syncLatch;
69
70 private TableName tableName;
71 private boolean skipTableStateCheck;
72 private UserGroupInformation user;
73
74 private Boolean traceEnabled = null;
75
76 public EnableTableProcedure() {
77 syncLatch = null;
78 }
79
80
81
82
83
84
85
86
87 public EnableTableProcedure(
88 final MasterProcedureEnv env,
89 final TableName tableName,
90 final boolean skipTableStateCheck) throws IOException {
91 this(env, tableName, skipTableStateCheck, null);
92 }
93
94
95
96
97
98
99
100
101 public EnableTableProcedure(
102 final MasterProcedureEnv env,
103 final TableName tableName,
104 final boolean skipTableStateCheck,
105 final ProcedurePrepareLatch syncLatch) throws IOException {
106 this.tableName = tableName;
107 this.skipTableStateCheck = skipTableStateCheck;
108 this.user = env.getRequestUser().getUGI();
109 this.setOwner(this.user.getShortUserName());
110
111
112
113
114
115
116
117
118
119 this.syncLatch = syncLatch;
120 }
121
122 @Override
123 protected Flow executeFromState(final MasterProcedureEnv env, final EnableTableState state)
124 throws InterruptedException {
125 if (isTraceEnabled()) {
126 LOG.trace(this + " execute state=" + state);
127 }
128
129 try {
130 switch (state) {
131 case ENABLE_TABLE_PREPARE:
132 if (prepareEnable(env)) {
133 setNextState(EnableTableState.ENABLE_TABLE_PRE_OPERATION);
134 } else {
135 assert isFailed() : "enable should have an exception here";
136 return Flow.NO_MORE_STATE;
137 }
138 break;
139 case ENABLE_TABLE_PRE_OPERATION:
140 preEnable(env, state);
141 setNextState(EnableTableState.ENABLE_TABLE_SET_ENABLING_TABLE_STATE);
142 break;
143 case ENABLE_TABLE_SET_ENABLING_TABLE_STATE:
144 setTableStateToEnabling(env, tableName);
145 setNextState(EnableTableState.ENABLE_TABLE_MARK_REGIONS_ONLINE);
146 break;
147 case ENABLE_TABLE_MARK_REGIONS_ONLINE:
148 markRegionsOnline(env, tableName, true);
149 setNextState(EnableTableState.ENABLE_TABLE_SET_ENABLED_TABLE_STATE);
150 break;
151 case ENABLE_TABLE_SET_ENABLED_TABLE_STATE:
152 setTableStateToEnabled(env, tableName);
153 setNextState(EnableTableState.ENABLE_TABLE_POST_OPERATION);
154 break;
155 case ENABLE_TABLE_POST_OPERATION:
156 postEnable(env, state);
157 return Flow.NO_MORE_STATE;
158 default:
159 throw new UnsupportedOperationException("unhandled state=" + state);
160 }
161 } catch (HBaseException|IOException e) {
162 LOG.error("Error trying to enable table=" + tableName + " state=" + state, e);
163 setFailure("master-enable-table", e);
164 }
165 return Flow.HAS_MORE_STATE;
166 }
167
168 @Override
169 protected void rollbackState(final MasterProcedureEnv env, final EnableTableState state)
170 throws IOException {
171 if (isTraceEnabled()) {
172 LOG.trace(this + " rollback state=" + state);
173 }
174 try {
175 switch (state) {
176 case ENABLE_TABLE_POST_OPERATION:
177
178 break;
179 case ENABLE_TABLE_SET_ENABLED_TABLE_STATE:
180 DisableTableProcedure.setTableStateToDisabling(env, tableName);
181 break;
182 case ENABLE_TABLE_MARK_REGIONS_ONLINE:
183 markRegionsOfflineDuringRecovery(env);
184 break;
185 case ENABLE_TABLE_SET_ENABLING_TABLE_STATE:
186 DisableTableProcedure.setTableStateToDisabled(env, tableName);
187 break;
188 case ENABLE_TABLE_PRE_OPERATION:
189
190 break;
191 case ENABLE_TABLE_PREPARE:
192
193
194 ProcedurePrepareLatch.releaseLatch(syncLatch, this);
195 break;
196 default:
197 throw new UnsupportedOperationException("unhandled state=" + state);
198 }
199 } catch (HBaseException e) {
200 LOG.warn("Failed enable table rollback attempt step=" + state + " table=" + tableName, e);
201 throw new IOException(e);
202 } catch (IOException e) {
203
204
205 LOG.warn("Failed enable table rollback attempt step=" + state + " table=" + tableName, e);
206 throw e;
207 }
208 }
209
210 @Override
211 protected EnableTableState getState(final int stateId) {
212 return EnableTableState.valueOf(stateId);
213 }
214
215 @Override
216 protected int getStateId(final EnableTableState state) {
217 return state.getNumber();
218 }
219
220 @Override
221 protected EnableTableState getInitialState() {
222 return EnableTableState.ENABLE_TABLE_PREPARE;
223 }
224
225 @Override
226 protected void setNextState(final EnableTableState state) {
227 if (aborted.get()) {
228 setAbortFailure("Enable-table", "abort requested");
229 } else {
230 super.setNextState(state);
231 }
232 }
233
234 @Override
235 public boolean abort(final MasterProcedureEnv env) {
236 aborted.set(true);
237 return true;
238 }
239
240 @Override
241 protected boolean acquireLock(final MasterProcedureEnv env) {
242 if (env.waitInitialized(this)) return false;
243 return env.getProcedureQueue().tryAcquireTableExclusiveLock(this, tableName);
244 }
245
246 @Override
247 protected void releaseLock(final MasterProcedureEnv env) {
248 env.getProcedureQueue().releaseTableExclusiveLock(this, tableName);
249 }
250
251 @Override
252 public void serializeStateData(final OutputStream stream) throws IOException {
253 super.serializeStateData(stream);
254
255 MasterProcedureProtos.EnableTableStateData.Builder enableTableMsg =
256 MasterProcedureProtos.EnableTableStateData.newBuilder()
257 .setUserInfo(MasterProcedureUtil.toProtoUserInfo(user))
258 .setTableName(ProtobufUtil.toProtoTableName(tableName))
259 .setSkipTableStateCheck(skipTableStateCheck);
260
261 enableTableMsg.build().writeDelimitedTo(stream);
262 }
263
264 @Override
265 public void deserializeStateData(final InputStream stream) throws IOException {
266 super.deserializeStateData(stream);
267
268 MasterProcedureProtos.EnableTableStateData enableTableMsg =
269 MasterProcedureProtos.EnableTableStateData.parseDelimitedFrom(stream);
270 user = MasterProcedureUtil.toUserInfo(enableTableMsg.getUserInfo());
271 tableName = ProtobufUtil.toTableName(enableTableMsg.getTableName());
272 skipTableStateCheck = enableTableMsg.getSkipTableStateCheck();
273 }
274
275 @Override
276 public void toStringClassDetails(StringBuilder sb) {
277 sb.append(getClass().getSimpleName());
278 sb.append(" (table=");
279 sb.append(tableName);
280 sb.append(")");
281 }
282
283 @Override
284 public TableName getTableName() {
285 return tableName;
286 }
287
288 @Override
289 public TableOperationType getTableOperationType() {
290 return TableOperationType.ENABLE;
291 }
292
293
294
295
296
297
298
299
300
301 private boolean prepareEnable(final MasterProcedureEnv env) throws IOException {
302 boolean canTableBeEnabled = true;
303
304
305 if (!MetaTableAccessor.tableExists(env.getMasterServices().getConnection(), tableName)) {
306 setFailure("master-enable-table", new TableNotFoundException(tableName));
307 canTableBeEnabled = false;
308 } else if (!skipTableStateCheck) {
309
310
311
312
313
314
315
316
317
318 TableStateManager tsm = env.getMasterServices().getAssignmentManager().getTableStateManager();
319 if (!tsm.isTableState(tableName, ZooKeeperProtos.Table.State.DISABLED)) {
320 LOG.info("Table " + tableName + " isn't disabled; skipping enable");
321 setFailure("master-enable-table", new TableNotDisabledException(this.tableName));
322 canTableBeEnabled = false;
323 }
324 }
325
326
327 ProcedurePrepareLatch.releaseLatch(syncLatch, this);
328
329 return canTableBeEnabled;
330 }
331
332
333
334
335
336
337
338
339 private void preEnable(final MasterProcedureEnv env, final EnableTableState state)
340 throws IOException, InterruptedException {
341 runCoprocessorAction(env, state);
342 }
343
344
345
346
347
348
349
350 protected static void setTableStateToEnabling(
351 final MasterProcedureEnv env,
352 final TableName tableName) throws HBaseException, IOException {
353
354 LOG.info("Attempting to enable the table " + tableName);
355 env.getMasterServices().getAssignmentManager().getTableStateManager().setTableState(
356 tableName,
357 ZooKeeperProtos.Table.State.ENABLING);
358 }
359
360
361
362
363
364
365
366
367 protected static void markRegionsOnline(
368 final MasterProcedureEnv env,
369 final TableName tableName,
370 final Boolean retryRequired) throws IOException {
371
372
373
374
375 int maxTry = (retryRequired ? 10 : 1);
376 boolean done = false;
377
378 do {
379 try {
380 done = markRegionsOnline(env, tableName);
381 if (done) {
382 break;
383 }
384 maxTry--;
385 } catch (Exception e) {
386 LOG.warn("Received exception while marking regions online. tries left: " + maxTry, e);
387 maxTry--;
388 if (maxTry > 0) {
389 continue;
390 }
391 throw e;
392 }
393 } while (maxTry > 0);
394
395 if (!done) {
396 LOG.warn("Some or all regions of the Table '" + tableName + "' were offline");
397 }
398 }
399
400
401
402
403
404
405
406
407 private static boolean markRegionsOnline(final MasterProcedureEnv env, final TableName tableName)
408 throws IOException {
409 final AssignmentManager assignmentManager = env.getMasterServices().getAssignmentManager();
410 final MasterServices masterServices = env.getMasterServices();
411 final ServerManager serverManager = masterServices.getServerManager();
412 boolean done = false;
413
414
415 List<Pair<HRegionInfo, ServerName>> tableRegionsAndLocations;
416
417 if (TableName.META_TABLE_NAME.equals(tableName)) {
418 tableRegionsAndLocations =
419 new MetaTableLocator().getMetaRegionsAndLocations(masterServices.getZooKeeper());
420 } else {
421 tableRegionsAndLocations =
422 MetaTableAccessor.getTableRegionsAndLocations(
423 masterServices.getZooKeeper(), masterServices.getConnection(), tableName, true);
424 }
425
426 int countOfRegionsInTable = tableRegionsAndLocations.size();
427 Map<HRegionInfo, ServerName> regionsToAssign =
428 regionsToAssignWithServerName(env, tableRegionsAndLocations);
429
430
431 List<HRegionInfo> unrecordedReplicas =
432 AssignmentManager.replicaRegionsNotRecordedInMeta(new HashSet<HRegionInfo>(
433 regionsToAssign.keySet()), masterServices);
434 Map<ServerName, List<HRegionInfo>> srvToUnassignedRegs =
435 assignmentManager.getBalancer().roundRobinAssignment(unrecordedReplicas,
436 serverManager.getOnlineServersList());
437 if (srvToUnassignedRegs != null) {
438 for (Map.Entry<ServerName, List<HRegionInfo>> entry : srvToUnassignedRegs.entrySet()) {
439 for (HRegionInfo h : entry.getValue()) {
440 regionsToAssign.put(h, entry.getKey());
441 }
442 }
443 }
444
445 int offlineRegionsCount = regionsToAssign.size();
446
447 LOG.info("Table '" + tableName + "' has " + countOfRegionsInTable + " regions, of which "
448 + offlineRegionsCount + " are offline.");
449 if (offlineRegionsCount == 0) {
450 return true;
451 }
452
453 List<ServerName> onlineServers = serverManager.createDestinationServersList();
454 Map<ServerName, List<HRegionInfo>> bulkPlan =
455 env.getMasterServices().getAssignmentManager().getBalancer()
456 .retainAssignment(regionsToAssign, onlineServers);
457 if (bulkPlan != null) {
458 LOG.info("Bulk assigning " + offlineRegionsCount + " region(s) across " + bulkPlan.size()
459 + " server(s), retainAssignment=true");
460
461 BulkAssigner ba = new GeneralBulkAssigner(masterServices, bulkPlan, assignmentManager, true);
462 try {
463 if (ba.bulkAssign()) {
464 done = true;
465 }
466 } catch (InterruptedException e) {
467 LOG.warn("Enable operation was interrupted when enabling table '" + tableName + "'");
468
469 Thread.currentThread().interrupt();
470 }
471 } else {
472 LOG.info("Balancer was unable to find suitable servers for table " + tableName
473 + ", leaving unassigned");
474 }
475 return done;
476 }
477
478
479
480
481
482 private void markRegionsOfflineDuringRecovery(final MasterProcedureEnv env) {
483 try {
484
485
486 DisableTableProcedure.markRegionsOffline(env, tableName, true);
487 } catch (Exception e) {
488 LOG.debug("Failed to offline all regions of table " + tableName + ". Ignoring", e);
489 }
490 }
491
492
493
494
495
496
497 protected static void setTableStateToEnabled(
498 final MasterProcedureEnv env,
499 final TableName tableName) throws HBaseException, IOException {
500
501 env.getMasterServices().getAssignmentManager().getTableStateManager().setTableState(
502 tableName,
503 ZooKeeperProtos.Table.State.ENABLED);
504 LOG.info("Table '" + tableName + "' was successfully enabled.");
505 }
506
507
508
509
510
511
512
513
514 private void postEnable(final MasterProcedureEnv env, final EnableTableState state)
515 throws IOException, InterruptedException {
516 runCoprocessorAction(env, state);
517 }
518
519
520
521
522
523
524 private Boolean isTraceEnabled() {
525 if (traceEnabled == null) {
526 traceEnabled = LOG.isTraceEnabled();
527 }
528 return traceEnabled;
529 }
530
531
532
533
534
535
536 private static Map<HRegionInfo, ServerName> regionsToAssignWithServerName(
537 final MasterProcedureEnv env,
538 final List<Pair<HRegionInfo, ServerName>> regionsInMeta) throws IOException {
539 Map<HRegionInfo, ServerName> regionsToAssign =
540 new HashMap<HRegionInfo, ServerName>(regionsInMeta.size());
541 RegionStates regionStates = env.getMasterServices().getAssignmentManager().getRegionStates();
542 for (Pair<HRegionInfo, ServerName> regionLocation : regionsInMeta) {
543 HRegionInfo hri = regionLocation.getFirst();
544 ServerName sn = regionLocation.getSecond();
545 if (regionStates.isRegionOffline(hri)) {
546 regionsToAssign.put(hri, sn);
547 } else {
548 if (LOG.isDebugEnabled()) {
549 LOG.debug("Skipping assign for the region " + hri + " during enable table "
550 + hri.getTable() + " because its already in tranition or assigned.");
551 }
552 }
553 }
554 return regionsToAssign;
555 }
556
557
558
559
560
561
562
563
564 private void runCoprocessorAction(final MasterProcedureEnv env, final EnableTableState state)
565 throws IOException, InterruptedException {
566 final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
567 if (cpHost != null) {
568 user.doAs(new PrivilegedExceptionAction<Void>() {
569 @Override
570 public Void run() throws Exception {
571 switch (state) {
572 case ENABLE_TABLE_PRE_OPERATION:
573 cpHost.preEnableTableHandler(getTableName());
574 break;
575 case ENABLE_TABLE_POST_OPERATION:
576 cpHost.postEnableTableHandler(getTableName());
577 break;
578 default:
579 throw new UnsupportedOperationException(this + " unhandled state=" + state);
580 }
581 return null;
582 }
583 });
584 }
585 }
586 }