擅长:python、mysql、java
<p><strong>wordcount的独立python脚本:使用contextmanager编写可重用的spark上下文</p>
<pre><code>"""SimpleApp.py"""
from contextlib import contextmanager
from pyspark import SparkContext
from pyspark import SparkConf
SPARK_MASTER='local'
SPARK_APP_NAME='Word Count'
SPARK_EXECUTOR_MEMORY='200m'
@contextmanager
def spark_manager():
conf = SparkConf().setMaster(SPARK_MASTER) \
.setAppName(SPARK_APP_NAME) \
.set("spark.executor.memory", SPARK_EXECUTOR_MEMORY)
spark_context = SparkContext(conf=conf)
try:
yield spark_context
finally:
spark_context.stop()
with spark_manager() as context:
File = "/home/ramisetty/sparkex/README.md" # Should be some file on your system
textFileRDD = context.textFile(File)
wordCounts = textFileRDD.flatMap(lambda line: line.split()).map(lambda word: (word, 1)).reduceByKey(lambda a, b: a+b)
wordCounts.saveAsTextFile("output")
print "WordCount - Done"
</code></pre>
<p><strong>启动:</strong></p>
<pre><code>/bin/spark-submit SimpleApp.py
</code></pre>