Skip to content

Commit e9a1e43

Browse files
lemirelemire
committed
Some experiments
1 parent cb7f39b commit e9a1e43

4 files changed

Lines changed: 24 additions & 10 deletions

File tree

‎src/main/java/me/lemire/integercompression/FastPFOR.java‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -192,14 +192,21 @@ private void encodePage(int[] in, IntWrapper inpos, int thissize,
192192
bitmap |= (1 << (k - 1));
193193
}
194194
out[tmpoutpos++] = bitmap;
195+
195196
for (int k = 1; k <= 32; ++k) {
196197
if (dataPointers[k] != 0) {
197198
out[tmpoutpos++] = dataPointers[k];// size
198-
for (int j = 0; j < dataPointers[k]; j += 32) {
199+
int j = 0;
200+
201+
for (; j + 31 < dataPointers[k]; j += 32) {
199202
BitPacking.fastpack(dataTobePacked[k],
200203
j, out, tmpoutpos, k);
201204
tmpoutpos += k;
202205
}
206+
int leftover = dataPointers[k] % 32;
207+
if(leftover > 0) {
208+
tmpoutpos = Util.pack(out, tmpoutpos, dataTobePacked[k], j, leftover, k);
209+
}
203210
}
204211
}
205212
outpos.set(tmpoutpos);
@@ -249,11 +256,17 @@ private void decodePage(int[] in, IntWrapper inpos, int[] out,
249256
if (dataTobePacked[k].length < size)
250257
dataTobePacked[k] = new int[Util
251258
.greatestMultiple(size + 31, 32)];
252-
for (int j = 0; j < size; j += 32) {
259+
int j = 0;
260+
261+
for (; j + 31 < size; j += 32) {
253262
BitPacking.fastunpack(in, inexcept,
254263
dataTobePacked[k], j, k);
255264
inexcept += k;
256265
}
266+
int leftover = size % 32;
267+
if(size > 0)
268+
inexcept = Util.unpack(in, inexcept, dataTobePacked[k], j, leftover, k);
269+
257270
}
258271
}
259272
Arrays.fill(dataPointers, 0);

‎src/main/java/me/lemire/integercompression/Util.java‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -109,19 +109,19 @@ protected static int packsize(int num, int b) {
109109
return (num + howmanyfit - 1) / howmanyfit;
110110
}
111111

112-
protected static int pack(int[] outputarray, int arraypos, int[] data,
112+
protected static int pack(int[] outputarray, int arraypos, int[] data, int datapos,
113113
int num, int b) {
114114
if (num == 0)
115115
return arraypos;
116116
if (b > 16) {
117-
System.arraycopy(data, 0, outputarray, arraypos, num);
117+
System.arraycopy(data, datapos, outputarray, arraypos, num);
118118
return num + arraypos;
119119
}
120120
for (int k = 0; k < packsize(num, b); ++k)
121121
outputarray[k + arraypos] = 0;
122122
int inwordpointer = 0;
123123
for (int k = 0; k < num; ++k) {
124-
outputarray[arraypos] |= (data[k] << inwordpointer);
124+
outputarray[arraypos] |= (data[k + datapos] << inwordpointer);
125125
inwordpointer += b;
126126
final int increment = ((inwordpointer + b - 1) >> 5);
127127
arraypos += increment;
@@ -130,7 +130,7 @@ protected static int pack(int[] outputarray, int arraypos, int[] data,
130130
return arraypos + (inwordpointer > 0 ? 1 : 0);
131131
}
132132

133-
protected static int unpack(int[] sourcearray, int arraypos, int[] data,
133+
protected static int unpack(int[] sourcearray, int arraypos, int[] data, int datapos,
134134
int num, int b) {
135135
if (b > 16) {
136136
System.arraycopy(sourcearray, arraypos, data, 0, num);
@@ -139,8 +139,7 @@ protected static int unpack(int[] sourcearray, int arraypos, int[] data,
139139
final int mask = (1 << b) - 1;
140140
int inwordpointer = 0;
141141
for (int k = 0; k < num; ++k) {
142-
data[k] = (sourcearray[arraypos] & mask);
143-
sourcearray[arraypos] >>>= b;
142+
data[k + datapos] = ((sourcearray[arraypos] >>> inwordpointer) & mask);
144143
inwordpointer += b;
145144
final int increment = ((inwordpointer + b - 1) >> 5);
146145
arraypos += increment;

‎src/main/java/me/lemire/integercompression/benchmarktools/BenchmarkSkippable.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,8 @@ private static int[][] generateTestData(ClusteredDataGenerator dataGen,
290290
}
291291

292292
static Object[] codecs = {
293+
new SkippableComposition(new FastPFOR(), new VariableByte()),
294+
293295
new SkippableIntegratedComposition(new IntegratedBinaryPacking(),
294296
new IntegratedVariableByte()), new JustCopy(), new VariableByte(),
295297

‎src/test/java/me/lemire/integercompression/TestUtils.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@ public void testPacking() {
2626
}
2727
for(int n = 0; n <= 32; ++n) {
2828
Arrays.fill(outputarray, 0);
29-
int howmany = Util.pack(outputarray, 0, data, n, b);
29+
int howmany = Util.pack(outputarray, 0, data,0, n, b);
3030
if(howmany != Util.packsize(n, b)) throw new RuntimeException("bug "+n+" "+b);
31-
Util.unpack(Arrays.copyOf(outputarray, howmany), 0, newdata, n, b);
31+
Util.unpack(Arrays.copyOf(outputarray, howmany), 0, newdata,0, n, b);
3232
for(int i = 0; i < n; ++i)
3333
if(newdata[i] != data[i]) {
3434
System.out.println(Arrays.toString(Arrays.copyOf(data, n)));

0 commit comments

Comments
 (0)