merged Dockerfile

cfregly · cfregly · commit 1799d275b100 · 2016-02-20T12:20:56.000-06:00
diff --git a/Dockerfile b/Dockerfile
@@ -225,7 +225,7 @@ RUN \
 
 # Airflow
  && cd ~ \
- && pip install airflow[devel] \
+ && pip install airflow \
 
 # Presto
  && cd ~ \
@@ -256,7 +256,7 @@ RUN \
  && unzip stanford-corenlp-full-2015-12-09.zip \
  && rm stanford-corenlp-full-2015-12-09.zip \
  && cd ~/pipeline/myapps/ml \
- && cp ~/stanford-corenlp-full-2015-12-09/stanford-corenlp-${CASSANDRA_VERSION}-models.jar lib/ \
+ && cp ~/stanford-corenlp-full-2015-12-09/stanford-corenlp-${STANFORD_CORENLP_VERSION}-models.jar lib/ \
  && sbt package \
 
 # Sbt Streaming
diff --git a/bin/docker-start-worker.sh b/bin/docker-start-worker.sh
@@ -0,0 +1 @@
+docker exec pipeline$1 bash && nohup /root/spark-1.6.0-bin-fluxcapacitor/sbin/start-slave.sh --cores 2 --memory 2048m --webui-port 6061 -h 172.17.0.$1 spark://172.17.0.1:7080
diff --git a/bin/docker-stop-workers.sh b/bin/docker-stop-workers.sh
@@ -0,0 +1 @@
+docker exec pipeline$1 nohup /root/spark-1.6.0-bin-fluxcapacitor/sbin/stop-slaves.sh
diff --git a/config/presto/catalog/cassandra.properties b/config/presto/catalog/cassandra.properties
@@ -0,0 +1,2 @@
+connector.name=cassandra
+cassandra.contact-points=127.0.0.1
diff --git a/config/presto/catalog/kafka.properties b/config/presto/catalog/kafka.properties
@@ -0,0 +1,3 @@
+connector.name=kafka
+kafka.table-names=item_ratings
+kafka.nodes=127.0.0.1:9092

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+docker exec pipeline$1 bash && nohup /root/spark-1.6.0-bin-fluxcapacitor/sbin/start-slave.sh --cores 2 --memory 2048m --webui-port 6061 -h 172.17.0.$1 spark://172.17.0.1:7080`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+docker exec pipeline$1 nohup /root/spark-1.6.0-bin-fluxcapacitor/sbin/stop-slaves.sh`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+connector.name=kafka`
	`2`	`+kafka.table-names=item_ratings`
	`3`	`+kafka.nodes=127.0.0.1:9092`