FilesExpand file tree

main

/

generate_csv_header.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

98 lines (77 loc) · 3.18 KB

main

/

generate_csv_header.py

Copy path

Top

File metadata and controls

• Code
• Blame

98 lines (77 loc) · 3.18 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

# this script generates data for the TPC-H dbgen

import os

from python_helpers import open_utf8

def get_csv_text(fpath, add_null_terminator=False):

with open(fpath, 'rb') as f:

text = bytearray(f.read())

result_text = ""

first = True

for byte in text:

if first:

result_text += str(byte)

else:

result_text += ", " + str(byte)

first = False

if add_null_terminator:

result_text += ", 0"

return result_text

def write_dir(dirname, varname):

files = os.listdir(dirname)

files.sort()

result = ""

aggregated_result = "const char *%s[] = {\n" % (varname,)

for fname in files:

file_varname = "%s_%s" % (varname, fname.split('.')[0])

result += "const uint8_t %s[] = {" % (file_varname,) + get_csv_text(os.path.join(dirname, fname), True) + "};\n"

aggregated_result += "\t(const char*) %s,\n" % (file_varname,)

aggregated_result = aggregated_result[:-2] + "\n};\n"

return result + aggregated_result

# ------------------------------------------- #

# ------------------------------------------- #

# ------------- TPC-H ------------ #

# ------------------------------------------- #

# ------------------------------------------- #

tpch_dir = 'extension/tpch/dbgen'

tpch_queries = os.path.join(tpch_dir, 'queries')

tpch_answers_sf001 = os.path.join(tpch_dir, 'answers', 'sf0.01')

tpch_answers_sf01 = os.path.join(tpch_dir, 'answers', 'sf0.1')

tpch_answers_sf1 = os.path.join(tpch_dir, 'answers', 'sf1')

tpch_header = os.path.join(tpch_dir, 'include', 'tpch_constants.hpp')

def create_tpch_header(tpch_dir):

result = """/* THIS FILE WAS AUTOMATICALLY GENERATED BY generate_csv_header.py */

#pragma once

const int TPCH_QUERIES_COUNT = 22;

"""

# write the queries

result += write_dir(tpch_queries, "TPCH_QUERIES")

result += write_dir(tpch_answers_sf001, "TPCH_ANSWERS_SF0_01")

result += write_dir(tpch_answers_sf01, "TPCH_ANSWERS_SF0_1")

result += write_dir(tpch_answers_sf1, "TPCH_ANSWERS_SF1")

with open_utf8(tpch_header, 'w+') as f:

f.write(result)

print(tpch_header)

create_tpch_header(tpch_dir)

# ------------------------------------------- #

# ------------------------------------------- #

# ------------- TPC-DS ------------ #

# ------------------------------------------- #

# ------------------------------------------- #

tpcds_dir = 'extension/tpcds/dsdgen'

tpcds_queries = os.path.join(tpcds_dir, 'queries')

tpcds_answers_sf001 = os.path.join(tpcds_dir, 'answers', 'sf0.01')

tpcds_answers_sf1 = os.path.join(tpcds_dir, 'answers', 'sf1')

tpcds_header = os.path.join(tpcds_dir, 'include', 'tpcds_constants.hpp')

def create_tpcds_header(tpch_dir):

result = """/* THIS FILE WAS AUTOMATICALLY GENERATED BY generate_csv_header.py */

#pragma once

const int TPCDS_QUERIES_COUNT = 99;

const int TPCDS_TABLE_COUNT = 24;

"""

# write the queries

result += write_dir(tpcds_queries, "TPCDS_QUERIES")

result += write_dir(tpcds_answers_sf001, "TPCDS_ANSWERS_SF0_01")

result += write_dir(tpcds_answers_sf1, "TPCDS_ANSWERS_SF1")

with open_utf8(tpcds_header, 'w+') as f:

f.write(result)

print(tpcds_header)

create_tpcds_header(tpcds_dir)