8000 Try and improve yarn cluster handling since its failing in travis (#200) · randomf/spark-testing-base@ac7111a · GitHub
[go: up one dir, main page]
Skip to content

Commit ac7111a

Browse files
holdenkholdenk
authored
Try and improve yarn cluster handling since its failing in travis (holdenk#200)
* Try and improve yarn cluster handling since its failing in travis * Manually run on precise? * Specify 3 nodes and also figure out which nodes are running * Actually launch the client * Only launch 2 nodes and ask for more memory right away. * Try limiting max memory for travis * Ok we need more memory, we can get it by asking travis to use sudo... * Just start xms and xmx at 4G * 3/3 * 2/2 * release notes update
1 parent edfd7a8 commit ac7111a

File tree

4 files changed

+30
-11
lines changed

4 files changed

+30
-11
lines changed

‎.travis.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
language: scala
2+
sudo: required
3+
dist: precise
24
cache:
35
directories:
46
- $HOME/.ivy2

‎RELEASE_NOTES.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
- Attempt to make it so that users doing SQL tests without Hive don't need the hive jars.
44
- Don't reset the SparkSession provider when in reuse mode.
55
- Add workaround for inaccessiable active context info in Spark 2.0
6+
- Upgrade to Hadoop 2.8.1 for mini cluster
7+
- Change build env after travis changes
68
# 0.7.2
79
- Add expiremental support to for reusing a SparkContext/Session accross multiple suites. For Spark 2.0+ only.
810
# 0.7.1

‎build.sbt

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ unmanagedSourceDirectories in Test := {
141141
}
142142

143143

144-
javaOptions ++= Seq("-Xms1G", "-Xmx3G", "-XX:MaxPermSize=2048M", "-XX:+CMSClassUnloadingEnabled")
144+
javaOptions ++= Seq("-Xms2G", "-Xmx2G", "-XX:MaxPermSize=2048M", "-XX:+CMSClassUnloadingEnabled")
145145

146146
// additional libraries
147147
libraryDependencies ++= Seq(
@@ -161,13 +161,13 @@ def excludeJavaxServlet(items: Seq[ModuleID]) =
161161
excludeFromAll(items, "javax.servlet", "servlet-api")
162162

163163
lazy val miniClusterDependencies = excludeJavaxServlet(Seq(
164-
"org.apache.hadoop" % "hadoop-hdfs" % "2.7.3" % "compile,test" classifier "" classifier "tests",
165-
"org.apache.hadoop" % "hadoop-common" % "2.7.3" % "compile,test" classifier "" classifier "tests" ,
166-
"org.apache.hadoop" % "hadoop-client" % "2.7.3" % "compile,test" classifier "" classifier "tests" ,
167-
"org.apache.hadoop" % "hadoop-mapreduce-client-jobclient" % "2.7.3" % "compile,test" classifier "" classifier "tests",
168-
"org.apache.hadoop" % "hadoop-yarn-server-tests" % "2.7.3" % "compile,test" classifier "" classifier "tests",
169-
"org.apache.hadoop" % "hadoop-yarn-server-web-proxy" % "2.7.3" % "compile,test" classifier "" classifier "tests",
170-
"org.apache.hadoop" % "hadoop-minicluster" % "2.7.3" % "compile,test"))
164+
"org.apache.hadoop" % "hadoop-hdfs" % "2.8.1" % "compile,test" classifier "" classifier "tests",
165+
"org.apache.hadoop" % "hadoop-common" % "2.8.1" % "compile,test" classifier "" classifier "tests" ,
166+
"org.apache.hadoop" % "hadoop-client" % "2.8.1" % "compile,test" classifier "" classifier "tests" ,
167+
"org.apache.hadoop" % "hadoop-mapreduce-client-jobclient" % "2.8.1" % "compile,test" classifier "" classifier "tests",
168+
"org.apache.hadoop" % "hadoop-yarn-server-tests" % "2.8.1" % "compile,test" classifier "" classifier "tests",
169+
"org.apache.hadoop" % "hadoop-yarn-server-web-proxy" % "2.8.1" % "compile,test" classifier "" classifier "tests",
170+
"org.apache.hadoop" % "hadoop-minicluster" % "2.8.1" % "compile,test"))
171171

172172
libraryDependencies ++= miniClusterDependencies
173173

‎src/main/1.3/scala/com/holdenkarau/spark/testing/YARNCluster.scala

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ import java.util.concurrent.TimeUnit
2424

2525
import com.google.common.base.Charsets.UTF_8
2626
import com.google.common.io.Files
27+
import org.apache.hadoop.yarn.client.api.YarnClient
28+
import org.apache.hadoop.yarn.api.records.NodeState
2729
import org.apache.hadoop.yarn.conf.YarnConfiguration
2830
import org.apache.hadoop.yarn.server.MiniYARNCluster
2931

@@ -68,19 +70,32 @@ trait YARNClusterLike {
6870
Files.write(LOG4J_CONF, logConfFile, UTF_8)
6971

7072
val yarnConf = new YarnConfiguration()
71-
yarnCluster = Some(new MiniYARNCluster(getClass().getName(), 1, 1, 1))
73+
// Disable the disk utilization check to avoid the test hanging when people's disks are
74+
// getting full.
75+
yarnConf.set("yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage",
76+
"100.0")
77+
78+
val nodes = 2
79+
yarnCluster = Some(new MiniYARNCluster(getClass().getName(), nodes, 1, 1))
7280
yarnCluster.foreach(_.init(yarnConf))
7381
yarnCluster.foreach(_.start())
7482

75-
val config = yarnCluster.map(_.getConfig())
83+
val config = yarnCluster.map(_.getConfig()).get
7684
val deadline = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(10)
77-
while (config.map(_.get(YarnConfiguration.RM_ADDRESS).split(":")(1)) == Some("0")) {
85+
while (config.get(YarnConfiguration.RM_ADDRESS).split(":")(1) == "0") {
7886
if (System.currentTimeMillis() > deadline) {
7987
throw new IllegalStateException("Timed out waiting for RM to come up.")
8088
}
8189
TimeUnit.MILLISECONDS.sleep(100)
8290
}
8391

92+
val yarnClient = YarnClient.createYarnClient()
93+
yarnClient.init(config)
94+
yarnClient.start()
95+
96+
val nodeReports = yarnClient.getNodeReports(NodeState.RUNNING)
97+
println(s"node reports in running: ${nodeReports}")
98+
8499
val props = setupSparkProperties()
85100
val propsFile = File.createTempFile("spark", ".properties", tempDir)
86101
val writer = new OutputStreamWriter(new FileOutputStream(propsFile), UTF_8)

0 commit comments

Comments
 (0)
0