FilesExpand file tree

 master

/

HeapFileEncoder.java

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

228 lines (201 loc) · 7.88 KB

 master

/

HeapFileEncoder.java

Copy path

Top

File metadata and controls

• Code
• Blame

228 lines (201 loc) · 7.88 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

package simpledb;

import java.io.*;

import java.util.ArrayList;

/**

* HeapFileEncoder reads a comma delimited text file or accepts

* an array of tuples and converts it to

* pages of binary data in the appropriate format for simpledb heap pages

* Pages are padded out to a specified length, and written consecutive in a

* data file.

*/

public class HeapFileEncoder {

/** Convert the specified tuple list (with only integer fields) into a binary

* page file. <br>

*

* The format of the output file will be as specified in HeapPage and

* HeapFile.

*

* @see HeapPage

* @see HeapFile

* @param tuples the tuples - a list of tuples, each represented by a list of integers that are

* the field values for that tuple.

* @param outFile The output file to write data to

* @param npagebytes The number of bytes per page in the output file

* @param numFields the number of fields in each input tuple

* @throws IOException if the temporary/output file can't be opened

*/

public static void convert(ArrayList<ArrayList<Integer>> tuples, File outFile, int npagebytes, int numFields) throws IOException {

File tempInput = File.createTempFile("tempTable", ".txt");

tempInput.deleteOnExit();

BufferedWriter bw = new BufferedWriter(new FileWriter(tempInput));

for (ArrayList<Integer> tuple : tuples) {

int writtenFields = 0;

for (Integer field : tuple) {

writtenFields++;

if (writtenFields > numFields) {

throw new RuntimeException("Tuple has more than " + numFields + " fields: (" +

Utility.listToString(tuple) + ")");

}

bw.write(String.valueOf(field));

if (writtenFields < numFields) {

bw.write(',');

}

}

bw.write('\n');

}

bw.close();

convert(tempInput, outFile, npagebytes, numFields);

}

public static void convert(File inFile, File outFile, int npagebytes,

int numFields) throws IOException {

Type[] ts = new Type[numFields];

for (int i = 0; i < ts.length; i++) {

ts[i] = Type.INT_TYPE;

}

convert(inFile,outFile,npagebytes,numFields,ts);

}

public static void convert(File inFile, File outFile, int npagebytes,

int numFields, Type[] typeAr)

throws IOException {

convert(inFile,outFile,npagebytes,numFields,typeAr,',');

}

/** Convert the specified input text file into a binary

* page file. <br>

* Assume format of the input file is (note that only integer fields are

* supported):<br>

* int,...,int\n<br>

* int,...,int\n<br>

* ...<br>

* where each row represents a tuple.<br>

* <p>

* The format of the output file will be as specified in HeapPage and

* HeapFile.

*

* @see HeapPage

* @see HeapFile

* @param inFile The input file to read data from

* @param outFile The output file to write data to

* @param npagebytes The number of bytes per page in the output file

* @param numFields the number of fields in each input line/output tuple

* @throws IOException if the input/output file can't be opened or a

* malformed input line is encountered

*/

public static void convert(File inFile, File outFile, int npagebytes,

int numFields, Type[] typeAr, char fieldSeparator)

throws IOException {

int nrecbytes = 0;

for (int i = 0; i < numFields ; i++) {

nrecbytes += typeAr[i].getLen();

}

System.out.println();

int nrecords = (npagebytes * 8) / (nrecbytes * 8 + 1); //floor comes for free

// per record, we need one bit; there are nrecords per page, so we need

// nrecords bits, i.e., ((nrecords/32)+1) integers.

int nheaderbytes = (nrecords / 8);

if (nheaderbytes * 8 < nrecords)

nheaderbytes++; //ceiling

int nheaderbits = nheaderbytes * 8;

BufferedReader br = new BufferedReader(new FileReader(inFile));

FileOutputStream os = new FileOutputStream(outFile);

// our numbers probably won't be much larger than 1024 digits

char buf[] = new char[1024];

//char buf[] = new char[10];

int curpos = 0;

int recordcount = 0;

int npages = 0;

int fieldNo = 0;

ByteArrayOutputStream headerBAOS = new ByteArrayOutputStream(nheaderbytes);

DataOutputStream headerStream = new DataOutputStream(headerBAOS);

ByteArrayOutputStream pageBAOS = new ByteArrayOutputStream(npagebytes);

DataOutputStream pageStream = new DataOutputStream(pageBAOS);

boolean done = false;

boolean first = true;

while (!done) {

int c = br.read();

// Ignore Windows/Notepad special line endings

if (c == '\r')

continue;

if (c == '\n') {

if (first)

continue;

recordcount++;

first = true;

} else

first = false;

if (c == fieldSeparator || c == '\n' || c == '\r') {

String s = new String(buf, 0, curpos);

if (typeAr[fieldNo] == Type.INT_TYPE) {

try {

pageStream.writeInt(Integer.parseInt(s.trim()));

} catch (NumberFormatException e) {

System.out.println ("BAD LINE : " + s);

}

}

else if (typeAr[fieldNo] == Type.STRING_TYPE) {

s = s.trim();

int overflow = Type.STRING_LEN - s.length();

if (overflow < 0) {

String news = s.substring(0,Type.STRING_LEN);

s = news;

}

pageStream.writeInt(s.length());

pageStream.writeBytes(s);

while (overflow-- > 0)

pageStream.write((byte)0);

}

curpos = 0;

if (c == '\n')

fieldNo = 0;

else

fieldNo++;

} else if (c == -1) {

done = true;

} else {

buf[curpos++] = (char)c;

continue;

}

// if we wrote a full page of records, or if we're done altogether,

// write out the header of the page.

//

// in the header, write a 1 for bits that correspond to records we've

// written and 0 for empty slots.

//

// when we're done, also flush the page to disk, but only if it has

// records on it. however, if this file is empty, do flush an empty

// page to disk.

if (recordcount >= nrecords

|| done && recordcount > 0

|| done && npages == 0) {

int i = 0;

byte headerbyte = 0;

for (i=0; i<nheaderbits; i++) {

if (i < recordcount)

headerbyte |= (1 << (i % 8));

if (((i+1) % 8) == 0) {

headerStream.writeByte(headerbyte);

headerbyte = 0;

}

}

if (i % 8 > 0)

headerStream.writeByte(headerbyte);

// pad the rest of the page with zeroes

for (i=0; i<(npagebytes - (recordcount * nrecbytes + nheaderbytes)); i++)

pageStream.writeByte(0);

// write header and body to file

headerStream.flush();

headerBAOS.writeTo(os);

pageStream.flush();

pageBAOS.writeTo(os);

// reset header and body for next page

headerBAOS = new ByteArrayOutputStream(nheaderbytes);

headerStream = new DataOutputStream(headerBAOS);

pageBAOS = new ByteArrayOutputStream(npagebytes);

pageStream = new DataOutputStream(pageBAOS);

recordcount = 0;

npages++;

}

}

br.close();

os.close();

}

}