FilesExpand file tree

 master

/

SortedWordCountSolution.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

18 lines (12 loc) · 623 Bytes

 master

/

SortedWordCountSolution.py

Copy path

Top

File metadata and controls

• Code
• Blame

18 lines (12 loc) · 623 Bytes

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

from pyspark import SparkContext, SparkConf

if __name__ == "__main__":

conf = SparkConf().setAppName("wordCounts").setMaster("local[*]")

sc = SparkContext(conf = conf)

lines = sc.textFile("in/word_count.text")

wordRdd = lines.flatMap(lambda line: line.split(" "))

wordPairRdd = wordRdd.map(lambda word: (word, 1))

wordToCountPairs = wordPairRdd.reduceByKey(lambda x, y: x + y)

sortedWordCountPairs = wordToCountPairs \

.sortBy(lambda wordCount: wordCount[1], ascending=False)

for word, count in sortedWordCountPairs.collect():

print("{} : {}".format(word, count))