Skip to content

Commit 476c70e

Browse files
author
Joe Nievelt
committed
Merge branch 'release/0.6.0'
2 parents 8f8843a + a0406b6 commit 476c70e

File tree

9 files changed

+257
-23
lines changed

9 files changed

+257
-23
lines changed

‎CHANGES.md

+6
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
# chill #
22

3+
### 0.6.0 ###
4+
* Add build instructions to readme and make InjectiveSerializer serializable #216
5+
* Build chill-scrooge for scala 2.11, too #219
6+
* Rewrite Java BitSet serializer to make it more efficient #220
7+
* Bijection 0.8.0, algebird 0.10.0, scala 2.10.5 #228
8+
39
### 0.5.2 ###
410
* Use new Travis CI infrastructure #210
511
* Optimizations for ConfiguredInstantiator. #213

‎README.md

+11-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,16 @@ Extensions for the [Kryo serialization library](https://github.com/EsotericSoftw
44
serializers and a set of classes to ease configuration of Kryo in systems like Hadoop, Storm,
55
Akka, etc.
66

7+
8+
### Buidling Chill
9+
10+
```bash
11+
./sbt
12+
> compile # to build chill
13+
> publishM2 # to publish chill to your local .m2 repo
14+
> publish-local # publish to local ivy repo.
15+
```
16+
717
Chill has a set of subprojects: chill-java, chill-hadoop, chill-storm and chill-scala. Other than
818
chill-scala, all these projects are written in Java so they are easy to use on any JVM platform.
919

@@ -149,7 +159,7 @@ Discussion occurs primarily on the [Chill mailing list](https://groups.google.co
149159

150160
## Maven
151161

152-
Chill modules are available on Maven Central. The current groupid and version for all modules is, respectively, `"com.twitter"` and `0.5.2` and each scala project is published for `2.10` and `2.11`. Search [search.maven.org](http://search.maven.org/#search%7Cga%7C1%7Cchill) when in doubt.
162+
Chill modules are available on Maven Central. The current groupid and version for all modules is, respectively, `"com.twitter"` and `0.6.0` and each scala project is published for `2.10` and `2.11`. Search [search.maven.org](http://search.maven.org/#search%7Cga%7C1%7Cchill) when in doubt.
153163

154164
## Authors
155165

‎chill-bijection/src/main/scala/com/twitter/chill/InjectiveSerializer.scala

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package com.twitter.chill
22

3+
import _root_.java.io.Serializable
4+
35
import com.twitter.bijection.Injection
46

57
/**
@@ -17,7 +19,7 @@ object InjectiveSerializer {
1719
new InjectiveSerializer(injection)
1820
}
1921

20-
class InjectiveSerializer[T] private (injection: Injection[T, Array[Byte]]) extends KSerializer[T] {
22+
class InjectiveSerializer[T] private (injection: Injection[T, Array[Byte]]) extends KSerializer[T] with Serializable {
2123
def write(kser: Kryo, out: Output, obj: T) {
2224
val bytes = injection(obj)
2325
out.writeInt(bytes.length, true)

‎chill-java/src/main/java/com/twitter/chill/java/BitSetSerializer.java

+57-8
Original file line numberDiff line numberDiff line change
@@ -17,39 +17,88 @@
1717
package com.twitter.chill.java;
1818

1919
import com.esotericsoftware.kryo.Kryo;
20+
import com.esotericsoftware.kryo.KryoException;
2021
import com.esotericsoftware.kryo.Serializer;
2122
import com.esotericsoftware.kryo.io.Input;
2223
import com.esotericsoftware.kryo.io.Output;
2324

2425
import com.twitter.chill.IKryoRegistrar;
2526
import com.twitter.chill.SingleRegistrar;
2627

28+
import java.io.Serializable;
29+
import java.lang.reflect.Constructor;
30+
import java.lang.reflect.Field;
31+
import java.lang.reflect.Method;
2732
import java.util.BitSet;
2833

29-
public class BitSetSerializer extends Serializer<BitSet> {
34+
public class BitSetSerializer extends Serializer<BitSet> implements Serializable {
3035

3136
static public IKryoRegistrar registrar() {
3237
return new SingleRegistrar(BitSet.class, new BitSetSerializer());
3338
}
3439

40+
private final static Field wordsField;
41+
private final static Constructor<BitSet> bitSetConstructor;
42+
private final static Method recalculateWordsInUseMethod;
43+
44+
static {
45+
try {
46+
wordsField = BitSet.class.getDeclaredField("words");
47+
wordsField.setAccessible(true);
48+
} catch (NoSuchFieldException e) {
49+
throw new KryoException("Error while getting field 'words' of bitSet", e);
50+
}
51+
try {
52+
bitSetConstructor = BitSet.class.getDeclaredConstructor(long[].class);
53+
bitSetConstructor.setAccessible(true);
54+
} catch (NoSuchMethodException e) {
55+
throw new KryoException("Unable to get BitSet(long[]) constructor", e);
56+
}
57+
try {
58+
recalculateWordsInUseMethod = BitSet.class.getDeclaredMethod("recalculateWordsInUse");
59+
recalculateWordsInUseMethod.setAccessible(true);
60+
} catch (NoSuchMethodException e) {
61+
throw new KryoException("Unable to get BitSet.recalculateWordsInUse() method", e);
62+
}
63+
}
64+
3565
@Override
3666
public void write(Kryo kryo, Output output, BitSet bitSet) {
37-
int len = bitSet.length();
38-
39-
output.writeInt(len, true);
67+
long words[] = null;
68+
// its sufficent to get only the 'words' field because
69+
// we can recompute the wordsInUse after deserialization
70+
try {
71+
words = (long[]) wordsField.get(bitSet);
72+
} catch (IllegalAccessException e) {
73+
throw new KryoException("Error while accessing field 'words' of bitSet", e);
74+
}
75+
output.writeInt(words.length, true);
4076

41-
for(int i = 0; i < len; i++) {
42-
output.writeBoolean(bitSet.get(i));
77+
for(int i = 0; i < words.length; i++) {
78+
output.writeLong(words[i]);
4379
}
4480
}
4581

4682
@Override
4783
public BitSet read(Kryo kryo, Input input, Class<BitSet> bitSetClass) {
4884
int len = input.readInt(true);
49-
BitSet ret = new BitSet(len);
85+
long[] target = new long[len];
5086

5187
for(int i = 0; i < len; i++) {
52-
ret.set(i, input.readBoolean());
88+
target[i] = input.readLong();
89+
}
90+
91+
BitSet ret = null;
92+
// call a private constructor: (the BitSet.valueOf() cTor is only available from Java 1.7)
93+
try {
94+
ret = bitSetConstructor.newInstance(target);
95+
} catch (ReflectiveOperationException e) {
96+
throw new KryoException("Unable to call BitSet(long[]) constructor", e);
97+
}
98+
try {
99+
recalculateWordsInUseMethod.invoke(ret);
100+
} catch (ReflectiveOperationException e) {
101+
throw new KryoException("Unable to call BitSet.recalculateWordsInUse() method", e);
53102
}
54103

55104
return ret;

‎chill-java/src/test/scala/com/twitter/chill/java/BitSetTest.scala

+111
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
package com.twitter.chill.java
2+
3+
import java.util
4+
5+
import com.esotericsoftware.kryo.Kryo
6+
import com.esotericsoftware.kryo.io.{Input, Output}
7+
import org.objenesis.strategy.StdInstantiatorStrategy
8+
import org.scalatest._
9+
10+
import scala.util.Random
11+
12+
class BitSetSpec extends WordSpec with MustMatchers {
13+
14+
implicit val kryo = new Kryo()
15+
16+
def rt[A](a: A)(implicit k: Kryo): A = {
17+
val out = new Output(1000, -1)
18+
k.writeClassAndObject(out, a.asInstanceOf[AnyRef])
19+
val in = new Input(out.toBytes)
20+
k.readClassAndObject(in).asInstanceOf[A]
21+
}
22+
23+
"A BitSetSerializer serializer" should {
24+
"handle BitSet" in {
25+
kryo.setInstantiatorStrategy(new StdInstantiatorStrategy)
26+
BitSetSerializer.registrar()(kryo)
27+
var simple = new util.BitSet(2048)
28+
simple.size() must be(2048)
29+
for (i <- 0 to 1337) {
30+
simple.set(i, true)
31+
}
32+
// we assume everything after 1337 to be false
33+
simple.get(1338) must equal(false)
34+
simple.get(2000) must equal(false)
35+
var dolly = rt(simple)
36+
simple = null // avoid accidental calls
37+
dolly.size() must be(2048)
38+
for (i <- 0 to 1337) {
39+
dolly.get(i) must be(true)
40+
}
41+
dolly.get(1338) must equal(false)
42+
dolly.get(2000) must equal(false)
43+
}
44+
45+
/**
46+
* My results:
47+
* The old serializer took 2886ms
48+
* The new serializer took 112ms
49+
* The old serializer needs 2051 bytes
50+
* The new serializer needs 258 bytes
51+
*/
52+
"handle a BitSet efficiently" in {
53+
val oldKryo = new Kryo()
54+
OldBitSetSerializer.registrar()(oldKryo)
55+
56+
val newKryo = new Kryo()
57+
BitSetSerializer.registrar()(newKryo)
58+
59+
val element = new util.BitSet(2048)
60+
val rnd = new Random()
61+
for (i <- 0 to 2048) {
62+
element.set(i, rnd.nextBoolean())
63+
}
64+
65+
// warmup In case anybody wants to see hotspot
66+
var lastBitSetFromOld : util.BitSet = null
67+
for(i <- 0 to 50000) {
68+
lastBitSetFromOld = rt(element)(oldKryo)
69+
}
70+
var start = System.currentTimeMillis()
71+
for(i <- 0 to 100000) {
72+
rt(element)(oldKryo)
73+
}
74+
println("The old serializer took "+(System.currentTimeMillis() - start)+"ms")
75+
76+
var lastBitSetFromNew : util.BitSet = null
77+
// warmup for the new kryo
78+
for(i <- 0 to 50000) {
79+
lastBitSetFromNew = rt(element)(newKryo)
80+
}
81+
// check for the three bitsets to be equal
82+
for (i <- 0 to 2048) {
83+
// original bitset against old serializer output
84+
element.get(i) must be(lastBitSetFromOld.get(i))
85+
86+
// original bitset against new serializer output
87+
element.get(i) must be(lastBitSetFromNew.get(i))
88+
}
89+
90+
91+
start = System.currentTimeMillis()
92+
for(i <- 0 to 100000) {
93+
rt(element)(newKryo)
94+
}
95+
println("The new serializer took "+(System.currentTimeMillis() - start)+"ms")
96+
97+
var out = new Output(1, -1)
98+
oldKryo.writeObject(out, element)
99+
out.flush()
100+
var oldBytes = out.total()
101+
println("The old serializer needs "+oldBytes+" bytes")
102+
out = new Output(1, -1)
103+
newKryo.writeObject(out, element)
104+
out.flush()
105+
var newBytes = out.total()
106+
println("The new serializer needs "+newBytes+" bytes")
107+
108+
oldBytes >= newBytes must be(true)
109+
}
110+
}
111+
}

‎chill-java/src/test/scala/com/twitter/chill/java/LocaleTest.scala

+3-3
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@ package com.twitter.chill.java
1818

1919
import org.scalatest._
2020

21-
import com.esotericsoftware.kryo.Kryo;
22-
import com.esotericsoftware.kryo.io.Input;
23-
import com.esotericsoftware.kryo.io.Output;
21+
import com.esotericsoftware.kryo.Kryo
22+
import com.esotericsoftware.kryo.io.Input
23+
import com.esotericsoftware.kryo.io.Output
2424

2525
import org.objenesis.strategy.StdInstantiatorStrategy
2626

‎chill-java/src/test/scala/com/twitter/chill/java/OldBitSetSerializer.java

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
Copyright 2013 Twitter, Inc.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package com.twitter.chill.java;
18+
19+
import com.esotericsoftware.kryo.Kryo;
20+
import com.esotericsoftware.kryo.Serializer;
21+
import com.esotericsoftware.kryo.io.Input;
22+
import com.esotericsoftware.kryo.io.Output;
23+
import com.twitter.chill.IKryoRegistrar;
24+
import com.twitter.chill.SingleRegistrar;
25+
26+
import java.util.BitSet;
27+
28+
public class OldBitSetSerializer extends Serializer<BitSet> {
29+
30+
static public IKryoRegistrar registrar() {
31+
return new SingleRegistrar(BitSet.class, new OldBitSetSerializer());
32+
}
33+
34+
@Override
35+
public void write(Kryo kryo, Output output, BitSet bitSet) {
36+
int len = bitSet.length();
37+
38+
output.writeInt(len, true);
39+
40+
for(int i = 0; i < len; i++) {
41+
output.writeBoolean(bitSet.get(i));
42+
}
43+
}
44+
45+
@Override
46+
public BitSet read(Kryo kryo, Input input, Class<BitSet> bitSetClass) {
47+
int len = input.readInt(true);
48+
BitSet ret = new BitSet(len);
49+
50+
for(int i = 0; i < len; i++) {
51+
ret.set(i, input.readBoolean());
52+
}
53+
54+
return ret;
55+
}
56+
}

‎chill-java/src/test/scala/com/twitter/chill/java/PriorityQueueTest.scala

+3-3
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@ package com.twitter.chill.java
1818

1919
import org.scalatest._
2020

21-
import com.esotericsoftware.kryo.Kryo;
22-
import com.esotericsoftware.kryo.io.Input;
23-
import com.esotericsoftware.kryo.io.Output;
21+
import com.esotericsoftware.kryo.Kryo
22+
import com.esotericsoftware.kryo.io.Input
23+
import com.esotericsoftware.kryo.io.Output
2424

2525
import org.objenesis.strategy.StdInstantiatorStrategy
2626

‎project/Build.scala

+7-7
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,10 @@ object ChillBuild extends Build {
2020

2121
val sharedSettings = Project.defaultSettings ++ mimaDefaultSettings ++ scalariformSettings ++ Seq(
2222

23-
version := "0.5.2",
23+
version := "0.6.0",
2424
organization := "com.twitter",
25-
scalaVersion := "2.10.4",
26-
crossScalaVersions := Seq("2.10.4", "2.11.5"),
25+
scalaVersion := "2.10.5",
26+
crossScalaVersions := Seq("2.10.5", "2.11.5"),
2727
scalacOptions ++= Seq("-unchecked", "-deprecation"),
2828
ScalariformKeys.preferences := formattingPreferences,
2929

@@ -126,7 +126,7 @@ object ChillBuild extends Build {
126126
.filterNot(unreleasedModules.contains(_))
127127
.map { s =>
128128
val suffix = if (javaOnly.contains(s)) "" else "_2.10"
129-
"com.twitter" % ("chill-" + s + suffix) % "0.5.2"
129+
"com.twitter" % ("chill-" + s + suffix) % "0.6.0"
130130
}
131131

132132
def module(name: String) = {
@@ -158,7 +158,7 @@ object ChillBuild extends Build {
158158

159159
lazy val chillBijection = module("bijection").settings(
160160
libraryDependencies ++= Seq(
161-
"com.twitter" %% "bijection-core" % "0.7.2"
161+
"com.twitter" %% "bijection-core" % "0.8.0"
162162
)
163163
).dependsOn(chill % "test->test;compile->compile")
164164

@@ -217,14 +217,14 @@ object ChillBuild extends Build {
217217

218218
lazy val chillAvro = module("avro").settings(
219219
libraryDependencies ++= Seq(
220-
"com.twitter" %% "bijection-avro" % "0.7.2",
220+
"com.twitter" %% "bijection-avro" % "0.8.0",
221221
"junit" % "junit" % "4.5" % "test"
222222
)
223223
).dependsOn(chill,chillJava, chillBijection)
224224

225225
lazy val chillAlgebird = module("algebird").settings(
226226
libraryDependencies ++= Seq(
227-
"com.twitter" %% "algebird-core" % "0.9.0"
227+
"com.twitter" %% "algebird-core" % "0.10.0"
228228
)
229229
).dependsOn(chill)
230230
}

0 commit comments

Comments
 (0)