FilesExpand file tree

master

/

HelloSpark.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

29 lines (22 loc) · 699 Bytes

master

/

HelloSpark.py

Copy path

Top

File metadata and controls

• Code
• Blame

29 lines (22 loc) · 699 Bytes

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

import sys

from pyspark.sql import *

from lib.logger import Log4j

from lib.utils import *

if __name__ == "__main__":

conf = get_spark_app_config()

spark = SparkSession \

.builder \

.appName("HelloSpark") \

.master("local[2]") \

.getOrCreate()

logger = Log4j(spark)

if len(sys.argv) != 2:

logger.error("Usage: HelloSpark <filename>")

sys.exit(-1)

logger.info("Starting HelloSpark")

survey_raw_df = load_survey_df(spark, sys.argv[1])

partitioned_survey_df = survey_raw_df.repartition(2)

count_df = count_by_country(partitioned_survey_df)

count_df.show()

logger.info("Finished HelloSpark")

spark.stop()