2.1 PySpark
2.2 Dask
2.3 Joblib
python
import dumbo
def mapper(key, value):
words = value.split()
for word in words:
yield word, 1
def reducer(key, values):
yield key, sum(values)
if __name__ == '__main__':
dumbo.run(mapper, reducer)
json
{
"hadoop_home": "/path/to/hadoop",
"hadoop_conf": "/path/to/hadoop/conf"
}