1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertTrue;
23
24 import java.io.IOException;
25 import java.util.ArrayList;
26 import java.util.List;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.hbase.client.HTable;
32 import org.apache.hadoop.hbase.client.RegionLocator;
33 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
34 import org.apache.hadoop.hbase.regionserver.HRegionServer;
35 import org.apache.hadoop.hbase.regionserver.Region;
36 import org.apache.hadoop.hbase.testclassification.MediumTests;
37 import org.apache.hadoop.hbase.util.Bytes;
38 import org.apache.hadoop.hbase.util.JVMClusterUtil;
39 import org.apache.hadoop.hbase.util.Threads;
40 import org.junit.Test;
41 import org.junit.experimental.categories.Category;
42
43
44
45
46
47 @Category(MediumTests.class)
48 public class TestGlobalMemStoreSize {
49 private static final Log LOG = LogFactory.getLog(TestGlobalMemStoreSize.class);
50 private static int regionServerNum = 4;
51 private static int regionNum = 16;
52
53 private static int totalRegionNum = regionNum+2;
54
55 private HBaseTestingUtility TEST_UTIL;
56 private MiniHBaseCluster cluster;
57
58
59
60
61
62
63 @Test
64 public void testGlobalMemStore() throws Exception {
65
66 LOG.info("Starting cluster");
67 Configuration conf = HBaseConfiguration.create();
68 TEST_UTIL = new HBaseTestingUtility(conf);
69 TEST_UTIL.startMiniCluster(1, regionServerNum);
70 cluster = TEST_UTIL.getHBaseCluster();
71 LOG.info("Waiting for active/ready master");
72 cluster.waitForActiveAndReadyMaster();
73
74
75 byte [] table = Bytes.toBytes("TestGlobalMemStoreSize");
76 byte [] family = Bytes.toBytes("family");
77 LOG.info("Creating table with " + regionNum + " regions");
78 HTable ht = TEST_UTIL.createMultiRegionTable(TableName.valueOf(table), family, regionNum);
79 int numRegions = -1;
80 try (RegionLocator r = ht.getRegionLocator()) {
81 numRegions = r.getStartKeys().length;
82 }
83 assertEquals(regionNum,numRegions);
84 waitForAllRegionsAssigned();
85
86 for (HRegionServer server : getOnlineRegionServers()) {
87 long globalMemStoreSize = 0;
88 for (HRegionInfo regionInfo :
89 ProtobufUtil.getOnlineRegions(null, server.getRSRpcServices())) {
90 globalMemStoreSize +=
91 server.getFromOnlineRegions(regionInfo.getEncodedName()).
92 getMemstoreSize();
93 }
94 assertEquals(server.getRegionServerAccounting().getGlobalMemstoreSize(),
95 globalMemStoreSize);
96 }
97
98
99 int i = 0;
100 for (HRegionServer server : getOnlineRegionServers()) {
101 LOG.info("Starting flushes on " + server.getServerName() +
102 ", size=" + server.getRegionServerAccounting().getGlobalMemstoreSize());
103
104 for (HRegionInfo regionInfo :
105 ProtobufUtil.getOnlineRegions(null, server.getRSRpcServices())) {
106 Region r = server.getFromOnlineRegions(regionInfo.getEncodedName());
107 flush(r, server);
108 }
109 LOG.info("Post flush on " + server.getServerName());
110 long now = System.currentTimeMillis();
111 long timeout = now + 1000;
112 while(server.getRegionServerAccounting().getGlobalMemstoreSize() != 0 &&
113 timeout < System.currentTimeMillis()) {
114 Threads.sleep(10);
115 }
116 long size = server.getRegionServerAccounting().getGlobalMemstoreSize();
117 if (size > 0) {
118
119
120 for (HRegionInfo regionInfo :
121 ProtobufUtil.getOnlineRegions(null, server.getRSRpcServices())) {
122 Region r = server.getFromOnlineRegions(regionInfo.getEncodedName());
123 long l = r.getMemstoreSize();
124 if (l > 0) {
125
126
127 assertTrue(regionInfo.isMetaRegion());
128 LOG.info(r.toString() + " " + l + ", reflushing");
129 r.flush(true);
130 }
131 }
132 }
133 size = server.getRegionServerAccounting().getGlobalMemstoreSize();
134 assertEquals("Server=" + server.getServerName() + ", i=" + i++, 0, size);
135 }
136
137 ht.close();
138 TEST_UTIL.shutdownMiniCluster();
139 }
140
141
142
143
144
145
146
147 private void flush(final Region r, final HRegionServer server)
148 throws IOException {
149 LOG.info("Flush " + r.toString() + " on " + server.getServerName() +
150 ", " + r.flush(true) + ", size=" +
151 server.getRegionServerAccounting().getGlobalMemstoreSize());
152 }
153
154 private List<HRegionServer> getOnlineRegionServers() {
155 List<HRegionServer> list = new ArrayList<HRegionServer>();
156 for (JVMClusterUtil.RegionServerThread rst :
157 cluster.getRegionServerThreads()) {
158 if (rst.getRegionServer().isOnline()) {
159 list.add(rst.getRegionServer());
160 }
161 }
162 return list;
163 }
164
165
166
167
168 private void waitForAllRegionsAssigned() throws IOException {
169 while (true) {
170 int regionCount = HBaseTestingUtility.getAllOnlineRegions(cluster).size();
171 if (regionCount >= totalRegionNum) break;
172 LOG.debug("Waiting for there to be "+ totalRegionNum
173 +" regions, but there are " + regionCount + " right now.");
174 try {
175 Thread.sleep(100);
176 } catch (InterruptedException e) {}
177 }
178 }
179 }