Merge branch 'release/0.3.5'

Author: johnynek

.travis.yml

@@ -1,4 +1,4 @@
 language: scala
 scala:
-  - 2.10.0
+  - 2.10.3
   - 2.9.3

CHANGES.md

@@ -1,5 +1,12 @@
 # chill #
 
+### 0.3.5
+* Add Serializers for scala SortedList and ListSet. https://github.com/twitter/chill/pull/152
+* Fix Range serialization (and remove broken subclass-serialization of Iterable, Seq, etc): https://github.com/twitter/chill/pull/154
+* Build and test chill-akka for 2.10: https://github.com/twitter/chill/pull/155
+* Add chill-thrift: https://github.com/twitter/chill/pull/156
+* Support JavaConverter-built classes: https://github.com/twitter/chill/pull/159
+
 ### 0.3.4
 * Bugfixes for Externalizer with looped object graphs https://github.com/twitter/chill/pull/143
 

README.md

@@ -147,7 +147,7 @@ Discussion occurs primarily on the [Chill mailing list](https://groups.google.co
 
 ## Maven
 
-Chill modules are available on Maven Central. The current groupid and version for all modules is, respectively, `"com.twitter"` and  `0.3.3`.
+Chill modules are available on Maven Central. The current groupid and version for all modules is, respectively, `"com.twitter"` and  `0.3.5`.
 
 Current published artifacts are
 

chill-java/src/main/java/com/twitter/chill/KryoInstantiator.java

@@ -47,7 +47,12 @@ public KryoInstantiator setReferences(final boolean ref) {
     return new KryoInstantiator() {
       public Kryo newKryo() {
         Kryo k = KryoInstantiator.this.newKryo();
-        k.setReferences(ref);
+        /**
+         * Kryo 2.17, used in storm, has this method returning void,
+         * 2.21 has it returning boolean.
+         * Try not to call the method if you don't need to.
+         */
+        if(k.getReferences() != ref) { k.setReferences(ref); }
         return k;
       }
     };
@@ -59,7 +64,10 @@ public KryoInstantiator setRegistrationRequired(final boolean req) {
     return new KryoInstantiator() {
       public Kryo newKryo() {
         Kryo k = KryoInstantiator.this.newKryo();
-        k.setRegistrationRequired(req);
+        /** Try to avoid calling this method if you don't need to.
+         * We've been burned by binary compatibility with Kryo
+         */
+        if(k.isRegistrationRequired() != req) { k.setRegistrationRequired(req); }
         return k;
       }
     };

chill-scala/src/main/scala/com/twitter/chill/ScalaKryoInstantiator.scala

@@ -18,6 +18,10 @@ package com.twitter.chill
 
 import scala.collection.immutable.{
   BitSet,
+  ListSet,
+  NumericRange,
+  Range,
+  SortedSet,
   ListMap,
   HashMap,
   Queue
@@ -39,6 +43,8 @@ import scala.util.matching.Regex
 import com.twitter.chill.java.PackageRegistrar
 import _root_.java.io.Serializable
 
+import scala.collection.JavaConverters._
+
 /** This class has a no-arg constructor, suitable for use with reflection instantiation
  * It has no registered serializers, just the standard Kryo configured for Kryo.
  */
@@ -83,6 +89,19 @@ class ScalaKryoInstantiator extends EmptyScalaKryoInstantiator {
 
 class ScalaCollectionsRegistrar extends IKryoRegistrar {
   def apply(newK: Kryo) {
+    // for binary compat this is here, but could be moved to RichKryo
+    def useField[T](cls: Class[T]) {
+      val fs = new com.esotericsoftware.kryo.serializers.FieldSerializer(newK, cls)
+      fs.setIgnoreSyntheticFields(false) // scala generates a lot of these attributes
+      newK.register(cls, fs)
+    }
+    // The wrappers are private classes:
+    useField(List(1, 2, 3).asJava.getClass)
+    useField(List(1, 2, 3).iterator.asJava.getClass)
+    useField(Map(1 -> 2, 4 -> 3).asJava.getClass)
+    useField(new _root_.java.util.ArrayList().asScala.getClass)
+    useField(new _root_.java.util.HashMap().asScala.getClass)
+
     /*
      * Note that subclass-based use: addDefaultSerializers, else: register
      * You should go from MOST specific, to least to specific when using
@@ -92,6 +111,7 @@ class ScalaCollectionsRegistrar extends IKryoRegistrar {
       // wrapper array is abstract
       .forSubclass[WrappedArray[Any]](new WrappedArraySerializer[Any])
       .forSubclass[BitSet](new BitSetSerializer)
+      .forSubclass[SortedSet[Any]](new SortedSetSerializer)
       .forClass[Some[Any]](new SomeSerializer[Any])
       .forClass[Left[Any, Any]](new LeftSerializer[Any, Any])
       .forClass[Right[Any, Any]](new RightSerializer[Any, Any])
@@ -102,20 +122,23 @@ class ScalaCollectionsRegistrar extends IKryoRegistrar {
       .forTraversableSubclass(Buffer.empty[Any], isImmutable = false)
       // Vector is a final class
       .forTraversableClass(Vector.empty[Any])
-      .forTraversableSubclass(IndexedSeq.empty[Any])
+      .forTraversableSubclass(ListSet.empty[Any])
       // specifically register small sets since Scala represents them differently
       .forConcreteTraversableClass(Set[Any]('a))
       .forConcreteTraversableClass(Set[Any]('a, 'b))
       .forConcreteTraversableClass(Set[Any]('a, 'b, 'c))
       .forConcreteTraversableClass(Set[Any]('a, 'b, 'c, 'd))
       .forConcreteTraversableClass(Set[Any]('a, 'b, 'c, 'd, 'e))
-      .forTraversableSubclass(Set.empty[Any])
       // specifically register small maps since Scala represents them differently
       .forConcreteTraversableClass(Map[Any, Any]('a -> 'a))
       .forConcreteTraversableClass(Map[Any, Any]('a -> 'a, 'b -> 'b))
       .forConcreteTraversableClass(Map[Any, Any]('a -> 'a, 'b -> 'b, 'c -> 'c))
       .forConcreteTraversableClass(Map[Any, Any]('a -> 'a, 'b -> 'b, 'c -> 'c, 'd -> 'd))
       .forConcreteTraversableClass(Map[Any, Any]('a -> 'a, 'b -> 'b, 'c -> 'c, 'd -> 'd, 'e -> 'e))
+      // The normal fields serializer works for ranges
+      .registerClasses(Seq(classOf[Range.Inclusive],
+                           classOf[NumericRange.Inclusive[_]],
+                           classOf[NumericRange.Exclusive[_]]))
       // Add some maps
       .forTraversableSubclass(ListMap.empty[Any,Any])
       .forTraversableSubclass(HashMap.empty[Any,Any])
@@ -128,12 +151,6 @@ class ScalaCollectionsRegistrar extends IKryoRegistrar {
       .forTraversableSubclass(MMap.empty[Any,Any], isImmutable = false)
       .forTraversableSubclass(MSet.empty[Any], isImmutable = false)
       .forTraversableSubclass(ListBuffer.empty[Any], isImmutable = false)
-      // This should be last, lots of things are seq/iterable/traversable
-      // These are questionable and might break things.
-      // rarely will you only expect an iterable/traversable on the reverse
-      .forTraversableSubclass(Seq.empty[Any])
-      .forTraversableSubclass(Iterable.empty[Any])
-      .forTraversableSubclass(Traversable.empty[Any])
     }
 }
 

chill-scala/src/main/scala/com/twitter/chill/SortedSetSerializer.scala

@@ -0,0 +1,52 @@
+/*
+Copyright 2012 Twitter, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package com.twitter.chill
+
+import scala.collection.immutable.SortedSet
+
+class SortedSetSerializer[T] extends KSerializer[SortedSet[T]] {
+  def write(kser: Kryo, out: Output, set: SortedSet[T]) {
+    //Write the size
+    out.writeInt(set.size, true)
+
+    // Write the ordering
+    kser.writeClassAndObject(out, set.ordering.asInstanceOf[AnyRef])
+    set.foreach { t  =>
+      val tRef = t.asInstanceOf[AnyRef]
+      kser.writeClassAndObject(out, tRef)
+      // After each intermediate object, flush
+      out.flush()
+    }
+  }
+
+  def read(kser: Kryo, in: Input, cls: Class[SortedSet[T]]): SortedSet[T] = {
+    val size = in.readInt(true)
+    val ordering = kser.readClassAndObject(in).asInstanceOf[Ordering[T]]
+
+    // Go ahead and be faster, and not as functional cool, and be mutable in here
+    var idx = 0
+    val builder = SortedSet.canBuildFrom[T](ordering)()
+    builder.sizeHint(size)
+
+    while (idx < size) {
+      val item = kser.readClassAndObject(in).asInstanceOf[T]
+      builder += item
+      idx += 1
+    }
+    builder.result()
+  }
+}

chill-scala/src/test/scala/com/twitter/chill/KryoSpec.scala

@@ -18,14 +18,13 @@ package com.twitter.chill
 
 import org.specs._
 
-import scala.collection.immutable.BitSet
-import scala.collection.immutable.ListMap
-import scala.collection.immutable.HashMap
-
+import scala.collection.immutable.{SortedSet, BitSet, ListSet, ListMap, HashMap}
 import scala.collection.mutable.{ArrayBuffer => MArrayBuffer, HashMap => MHashMap}
 import _root_.java.util.PriorityQueue
 import _root_.java.util.Locale
 import scala.collection.mutable
+import scala.collection.JavaConverters._
+
 /*
 * This is just a test case for Kryo to deal with. It should
 * be outside KryoSpec, otherwise the enclosing class, KryoSpec
@@ -49,6 +48,8 @@ trait ExampleUsingSelf { self =>
   def addOne = new ExampleUsingSelf {override def count=self.count+1}
 }
 
+case class Foo(m1: Map[String, Int], m2: Map[String, Seq[String]])
+
 class KryoSpec extends Specification with BaseProperties {
 
   noDetailedDiffs() //Fixes issue for scala 2.9
@@ -60,6 +61,7 @@ class KryoSpec extends Specification with BaseProperties {
       val test = List(1,2,"hey",(1,2),
                       ("hey","you"),
                       ("slightly", 1L, "longer", 42, "tuple"),
+                      Foo(Map("1" -> 1), Map("1" -> Seq("foo.com"))),
                       Map(1->2,4->5),
                       0 to 100,
                       (0 to 42).toList, Seq(1,100,1000),
@@ -70,6 +72,8 @@ class KryoSpec extends Specification with BaseProperties {
                       MArrayBuffer(1,2,3,4,5),
                       List(Some(MHashMap(1->1, 2->2)), None, Some(MHashMap(3->4))),
                       Set(1,2,3,4,10),
+                      SortedSet[Long](),
+                      SortedSet(1L, 2L, 3L, 4L),
                       BitSet(),
                       BitSet((0 until 1000).map{ x : Int => x*x } : _*),
                       ListMap("good" -> 0.5, "bad" -> -1.0),
@@ -83,6 +87,10 @@ class KryoSpec extends Specification with BaseProperties {
                       Vector(1,2,3,4,5),
                       TestValMap(null),
                       Some("junk"),
+                      List(1, 2, 3).asJava,
+                      Map("hey" -> 1, "you" -> 2).asJava,
+                      new _root_.java.util.ArrayList(Seq(1, 2, 3).asJava).asScala,
+                      new _root_.java.util.HashMap[Int,Int](Map(1 -> 2, 3 -> 4).asJava).asScala,
                       (),
                       'hai)
         .asInstanceOf[List[AnyRef]]
@@ -92,6 +100,22 @@ class KryoSpec extends Specification with BaseProperties {
         serdeser must be_==(orig)
       }
     }
+    "round trip a SortedSet" in {
+      val a = SortedSet[Long]() // Test empty SortedSet
+      val b = SortedSet[Int](1,2) // Test small SortedSet
+      val c = SortedSet[Int](1,2,3,4,6,7,8,9,10)(Ordering.fromLessThan((x, y) => x > y)) // Test with different ordering
+      rt(a) must be_==(a)
+      rt(b) must be_==(b)
+      (rt(c) + 5) must be_==(c + 5)
+    }
+    "round trip a ListSet" in {
+      val a = ListSet[Long]() // Test empty SortedSet
+      val b = ListSet[Int](1,2) // Test small ListSet
+      val c = ListSet[Int](1,2,3,4,6,7,8,9,10)
+      rt(a) must be_==(a)
+      rt(b) must be_==(b)
+      (rt(c)) must be_==(c)
+    }
     "handle trait with reference of self" in {
       var a= new ExampleUsingSelf{}
       var b=rt(a.addOne)
@@ -270,5 +294,20 @@ class KryoSpec extends Specification with BaseProperties {
       val qrlist = toList(qr)
       toList(rt(qr)) must be_==(qrlist)
     }
+    "Ranges should be fixed size" in {
+      val MAX_RANGE_SIZE = 188 // what seems to be needed.
+      serialize((1 to 10000)).size must be_<(MAX_RANGE_SIZE) // some fixed size
+      serialize((1 to 10000 by 2)).size must be_<(MAX_RANGE_SIZE) // some fixed size
+      serialize((1 until 10000)).size must be_<(MAX_RANGE_SIZE) // some fixed size
+      serialize((1 until 10000 by 2)).size must be_<(MAX_RANGE_SIZE) // some fixed size
+      serialize((1L to 10000L)).size must be_<(MAX_RANGE_SIZE) // some fixed size
+      serialize((1L to 10000L by 2L)).size must be_<(MAX_RANGE_SIZE) // some fixed size
+      serialize((1L until 10000L)).size must be_<(MAX_RANGE_SIZE) // some fixed size
+      serialize((1L until 10000L by 2L)).size must be_<(MAX_RANGE_SIZE) // some fixed size
+      serialize((1.0 to 10000.0)).size must be_<(MAX_RANGE_SIZE) // some fixed size
+      serialize((1.0 to 10000.0 by 2.0)).size must be_<(MAX_RANGE_SIZE) // some fixed size
+      serialize((1.0 until 10000.0)).size must be_<(MAX_RANGE_SIZE) // some fixed size
+      serialize((1.0 until 10000.0 by 2.0)).size must be_<(MAX_RANGE_SIZE) // some fixed size
+    }
   }
 }

chill-thrift/src/main/java/com/twitter/chill/thrift/TBaseSerializer.java

@@ -0,0 +1,69 @@
+/*
+Copyright 2013 Twitter, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package com.twitter.chill.thrift;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+
+import org.apache.thrift.TBase;
+import org.apache.thrift.TDeserializer;
+import org.apache.thrift.TException;
+import org.apache.thrift.TSerializer;
+
+/**
+ * Kryo serializer for Thrift instances.
+ *
+ * Note that this class is not thread-safe. (Kryo itself is not thread
+ * safe, so this shouldn't be a concern.)
+ *
+ * Use this with
+ * addDefaultSerializer(TBase.class, TBaseSerializer.class)
+ * It still helps to .register your instances so the full class name
+ * does not need to be written.
+ */
+public class TBaseSerializer extends Serializer<TBase> {
+  private final TSerializer serializer  = new TSerializer();
+  private final TDeserializer deserializer  = new TDeserializer();
+
+  @Override
+  public void write(Kryo kryo, Output output, TBase tBase) {
+    try {
+      byte[] serThrift = serializer.serialize(tBase);
+      output.writeInt(serThrift.length, true);
+      output.writeBytes(serThrift);
+    } catch (TException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public TBase read(Kryo kryo, Input input, Class<TBase> tBaseClass) {
+    try {
+      TBase prototype = tBaseClass.newInstance();
+      int tSize = input.readInt(true);
+      byte[] barr = new byte[tSize];
+      input.readBytes(barr);
+      deserializer.deserialize(prototype, barr);
+      return prototype;
+    } catch (Exception e) {
+      throw new RuntimeException("Could not create " + tBaseClass, e);
+    }
+  }
+}
+