FilesExpand file tree

 develop

/

example.java

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

177 lines (161 loc) · 7 KB

 develop

/

example.java

Top

File metadata and controls

• Code
• Blame

177 lines (161 loc) · 7 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

import me.lemire.integercompression.*;

import java.util.*;

public class example {

public static void main(String[] args) {

unsortedExample();

basicExample();

advancedExample();

}

public static void basicExample() {

int[] data = new int[2342351];

System.out.println("Compressing "+data.length+" integers in one go");

// data should be sorted for best

//results

for(int k = 0; k < data.length; ++k)

data[k] = k;

// Very important: the data is in sorted order!!! If not, you

// will get very poor compression with IntegratedBinaryPacking,

// you should use another CODEC.

// next we compose a CODEC. Most of the processing

// will be done with binary packing, and leftovers will

// be processed using variable byte

IntegratedIntegerCODEC codec = new

IntegratedComposition(

new IntegratedBinaryPacking(),

new IntegratedVariableByte());

// output vector should be large enough...

int [] compressed = new int[data.length];

// compressed might not be large enough in some cases

// if you get java.lang.ArrayIndexOutOfBoundsException, try

// allocating more memory

/**

*

* compressing

*

*/

IntWrapper inputoffset = new IntWrapper(0);

IntWrapper outputoffset = new IntWrapper(0);

codec.compress(data,inputoffset,data.length,compressed,outputoffset);

// got it!

// inputoffset should be at data.length but outputoffset tells

// us where we are...

System.out.println("compressed from "+data.length*4/1024+"KB to "+outputoffset.intValue()*4/1024+"KB");

// we can repack the data: (optional)

compressed = Arrays.copyOf(compressed,outputoffset.intValue());

/**

*

* now uncompressing

*

*/

int[] recovered = new int[data.length];

IntWrapper recoffset = new IntWrapper(0);

codec.uncompress(compressed,new IntWrapper(0),compressed.length,recovered,recoffset);

if(Arrays.equals(data,recovered))

System.out.println("data is recovered without loss");

else

throw new RuntimeException("bug"); // could use assert

System.out.println();

}

/**

* This is an example to show you can compress unsorted integers

* as long as most are small.

*/

public static void unsortedExample() {

final int N = 1333333;

int[] data = new int[N];

// initialize the data (most will be small

for(int k = 0; k < N; k+=1) data[k] = 3;

// throw some larger values

for(int k = 0; k < N; k+=5) data[k] = 100;

for(int k = 0; k < N; k+=533) data[k] = 10000;

int[] compressed = new int [N];// could need more

IntegerCODEC codec = new

Composition(

new FastPFOR(),

new VariableByte());

// compressing

IntWrapper inputoffset = new IntWrapper(0);

IntWrapper outputoffset = new IntWrapper(0);

codec.compress(data,inputoffset,data.length,compressed,outputoffset);

System.out.println("compressed unsorted integers from "+data.length*4/1024+"KB to "+outputoffset.intValue()*4/1024+"KB");

// we can repack the data: (optional)

compressed = Arrays.copyOf(compressed,outputoffset.intValue());

int[] recovered = new int[N];

IntWrapper recoffset = new IntWrapper(0);

codec.uncompress(compressed,new IntWrapper(0),compressed.length,recovered,recoffset);

if(Arrays.equals(data,recovered))

System.out.println("data is recovered without loss");

else

throw new RuntimeException("bug"); // could use assert

System.out.println();

}

/**

* This is like the basic example, but we

* show how to process larger arrays in chunks.

*

* Some of this code was written by Pavel Klinov.

*/

public static void advancedExample() {

int TotalSize = 2342351; // some arbitrary number

int ChunkSize = 16384; // size of each chunk, choose a multiple of 128

System.out.println("Compressing "+TotalSize+" integers using chunks of "+ChunkSize+" integers ("+ChunkSize*4/1024+"KB)");

System.out.println("(It is often better for applications to work in chunks fitting in CPU cache.)");

int[] data = new int[TotalSize];

// data should be sorted for best

//results

for(int k = 0; k < data.length; ++k)

data[k] = k;

// next we compose a CODEC. Most of the processing

// will be done with binary packing, and leftovers will

// be processed using variable byte, using variable byte

// only for the last chunk!

IntegratedIntegerCODEC regularcodec = new

IntegratedBinaryPacking();

IntegratedVariableByte ivb = new IntegratedVariableByte();

IntegratedIntegerCODEC lastcodec = new

IntegratedComposition(regularcodec,ivb);

// output vector should be large enough...

int [] compressed = new int[TotalSize];

/**

*

* compressing

*

*/

IntWrapper inputoffset = new IntWrapper(0);

IntWrapper outputoffset = new IntWrapper(0);

for(int k = 0; k < TotalSize / ChunkSize; ++k)

regularcodec.compress(data,inputoffset,ChunkSize,compressed,outputoffset);

lastcodec.compress(data, inputoffset, TotalSize % ChunkSize, compressed, outputoffset);

// got it!

// inputoffset should be at data.length but outputoffset tells

// us where we are...

System.out.println("compressed from "+data.length*4/1024+"KB to "+outputoffset.intValue()*4/1024+"KB");

// we can repack the data:

compressed = Arrays.copyOf(compressed,outputoffset.intValue());

/**

*

* now uncompressing

*

* We are *not* assuming that the original array length is known, however

* we assume that the chunk size (ChunkSize) is known.

*

*/

int[] recovered = new int[ChunkSize];

IntWrapper compoff = new IntWrapper(0);

IntWrapper recoffset;

int currentpos = 0;

while(compoff.get()<compressed.length) {

recoffset = new IntWrapper(0);

regularcodec.uncompress(compressed,compoff,compressed.length - compoff.get(),recovered,recoffset);

if(recoffset.get() < ChunkSize) {// last chunk detected

ivb.uncompress(compressed,compoff,compressed.length - compoff.get(),recovered,recoffset);

}

for(int i = 0; i < recoffset.get(); ++i) {

if(data[currentpos+i] != recovered[i]) throw new RuntimeException("bug"); // could use assert

}

currentpos += recoffset.get();

}

System.out.println("data is recovered without loss");

System.out.println();

}

}