Skip to content

Commit fa8de69

Browse files
ianocianoc
committed
Merge branch 'release/0.5.1'
2 parents 8d47172 + 8432ace commit fa8de69

File tree

8 files changed

+132
-42
lines changed

8 files changed

+132
-42
lines changed

‎CHANGES.md

+7
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
# chill #
22

3+
### 0.5.1
4+
* reverse Config contains logic: https://github.com/twitter/chill/pull/205
5+
* fix setConf: https://github.com/twitter/chill/pull/204
6+
* fix default constructor for kryo serialization: https://github.com/twitter/chill/pull/203
7+
* Switched Chill Avro to use ClassTags, added Java unit tests: https://github.com/twitter/chill/pull/200
8+
* Enable cross compilation for chill-avro: https://github.com/twitter/chill/pull/202
9+
310
### 0.5.0
411
* Move to211: https://github.com/twitter/chill/pull/197
512
* Make 2.10.4 the default, move to scalatest: https://github.com/twitter/chill/pull/196

‎README.md

+1-15
Original file line numberDiff line numberDiff line change
@@ -147,21 +147,7 @@ Discussion occurs primarily on the [Chill mailing list](https://groups.google.co
147147

148148
## Maven
149149

150-
Chill modules are available on Maven Central. The current groupid and version for all modules is, respectively, `"com.twitter"` and `0.3.6`.
151-
152-
Current published artifacts are
153-
154-
* `chill-java`
155-
* `chill-storm`
156-
* `chill-hadoop`
157-
* `chill_2.9.3`
158-
* `chill_2.10`
159-
* `chill-bijection_2.9.3`
160-
* `chill-bijection_2.10`
161-
* `chill-akka_2.9.3`
162-
* `chill-akka_2.10`
163-
164-
The suffix denotes the scala version.
150+
Chill modules are available on Maven Central. The current groupid and version for all modules is, respectively, `"com.twitter"` and `0.5.1` and each scala project is published for `2.10` and `2.11`. Search [search.maven.org](http://search.maven.org/#search%7Cga%7C1%7Cchill) when in doubt.
165151

166152
## Authors
167153

‎chill-avro/src/main/scala/com/twitter/chill/avro/AvroSerializer.scala

+9-8
Original file line numberDiff line numberDiff line change
@@ -14,38 +14,39 @@ limitations under the License.
1414
*/
1515
package com.twitter.chill.avro
1616

17-
import org.apache.avro.specific.SpecificRecordBase
18-
import com.twitter.chill.{ InjectiveSerializer, KSerializer }
17+
import com.twitter.bijection.Injection
1918
import com.twitter.bijection.avro.{ GenericAvroCodecs, SpecificAvroCodecs }
19+
import com.twitter.chill.{ InjectiveSerializer, KSerializer }
2020
import org.apache.avro.Schema
21-
import com.twitter.bijection.Injection
22-
import org.apache.avro.generic.GenericData.Record
2321
import org.apache.avro.generic.GenericRecord
22+
import org.apache.avro.specific.SpecificRecordBase
23+
24+
import scala.reflect.ClassTag
2425

2526
/**
2627
* @author Mansur Ashraf
2728
* @since 2/9/14.
2829
*/
2930
object AvroSerializer {
3031

31-
def SpecificRecordSerializer[T <: SpecificRecordBase: Manifest]: KSerializer[T] = {
32+
def SpecificRecordSerializer[T <: SpecificRecordBase: ClassTag]: KSerializer[T] = {
3233
implicit val inj = SpecificAvroCodecs[T]
3334
InjectiveSerializer.asKryo
3435
}
3536

36-
def SpecificRecordBinarySerializer[T <: SpecificRecordBase: Manifest]: KSerializer[T] = {
37+
def SpecificRecordBinarySerializer[T <: SpecificRecordBase: ClassTag]: KSerializer[T] = {
3738
implicit val inj = SpecificAvroCodecs.toBinary[T]
3839
InjectiveSerializer.asKryo
3940
}
4041

41-
def SpecificRecordJsonSerializer[T <: SpecificRecordBase: Manifest](schema: Schema): KSerializer[T] = {
42+
def SpecificRecordJsonSerializer[T <: SpecificRecordBase: ClassTag](schema: Schema): KSerializer[T] = {
4243
import com.twitter.bijection.StringCodec.utf8
4344
implicit val inj = SpecificAvroCodecs.toJson[T](schema)
4445
implicit val avroToArray = Injection.connect[T, String, Array[Byte]]
4546
InjectiveSerializer.asKryo
4647
}
4748

48-
def GenericRecordSerializer[T <: GenericRecord: Manifest](schema: Schema = null): KSerializer[T] = {
49+
def GenericRecordSerializer[T <: GenericRecord: ClassTag](schema: Schema = null): KSerializer[T] = {
4950
implicit val inj = GenericAvroCodecs[T](schema)
5051
InjectiveSerializer.asKryo
5152
}

‎chill-avro/src/test/java/com/twitter/chill/avro/AvroSerializerJavaTest.java

+92
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
package com.twitter.chill.avro;
2+
3+
import avro.FiscalRecord;
4+
import com.esotericsoftware.kryo.Kryo;
5+
import com.esotericsoftware.kryo.Serializer;
6+
import com.twitter.chill.KryoInstantiator;
7+
import com.twitter.chill.KryoPool;
8+
import org.apache.avro.Schema;
9+
import org.apache.avro.SchemaBuilder;
10+
import org.apache.avro.generic.GenericData;
11+
import org.apache.avro.generic.GenericRecordBuilder;
12+
import org.junit.Before;
13+
import org.junit.Test;
14+
import org.objenesis.strategy.StdInstantiatorStrategy;
15+
import scala.reflect.ClassTag;
16+
17+
import static org.junit.Assert.assertEquals;
18+
19+
public class AvroSerializerJavaTest {
20+
21+
private Schema schema;
22+
private GenericData.Record user;
23+
private FiscalRecord fiscalRecord;
24+
25+
@Before
26+
public void setUp() throws Exception {
27+
28+
schema = SchemaBuilder
29+
.record("person")
30+
.fields()
31+
.name("name").type().stringType().noDefault()
32+
.name("ID").type().intType().noDefault()
33+
.endRecord();
34+
35+
36+
user = new GenericRecordBuilder(schema)
37+
.set("name", "Jeff")
38+
.set("ID", 1)
39+
.build();
40+
41+
42+
fiscalRecord = FiscalRecord.newBuilder().setCalendarDate("2012-01-01").setFiscalWeek(1).setFiscalYear(2012).build();
43+
44+
}
45+
46+
public <T> KryoPool getKryo(final ClassTag<T> tag, final Serializer<T> serializer){
47+
KryoInstantiator kryoInstantiator = new KryoInstantiator() {
48+
public Kryo newKryo() {
49+
Kryo k =super.newKryo();
50+
k.setInstantiatorStrategy(new StdInstantiatorStrategy());
51+
k.register(tag.runtimeClass(), serializer);
52+
return k;
53+
}
54+
};
55+
56+
return KryoPool.withByteArrayOutputStream(1, kryoInstantiator);
57+
}
58+
@Test
59+
public void testSpecificRecordSerializer() throws Exception {
60+
ClassTag<FiscalRecord> tag = getClassTag(FiscalRecord.class);
61+
KryoPool kryo = getKryo(tag, AvroSerializer$.MODULE$.SpecificRecordSerializer(tag));
62+
byte[] bytes = kryo.toBytesWithClass(fiscalRecord);
63+
FiscalRecord result = (FiscalRecord) kryo.fromBytes(bytes);
64+
assertEquals(fiscalRecord,result);
65+
}
66+
67+
@Test
68+
public void SpecificRecordBinarySerializer() throws Exception {
69+
ClassTag<FiscalRecord> tag = getClassTag(FiscalRecord.class);
70+
KryoPool kryo = getKryo(tag, AvroSerializer$.MODULE$.SpecificRecordBinarySerializer(tag));
71+
byte[] bytes = kryo.toBytesWithClass(fiscalRecord);
72+
FiscalRecord result = (FiscalRecord) kryo.fromBytes(bytes);
73+
assertEquals(fiscalRecord,result);
74+
}
75+
76+
77+
@Test
78+
public void testGenericRecord() throws Exception {
79+
ClassTag<GenericData.Record> tag = getClassTag(GenericData.Record.class);
80+
KryoPool kryo = getKryo(tag, AvroSerializer$.MODULE$.GenericRecordSerializer(schema,tag));
81+
byte[] userBytes = kryo.toBytesWithClass(user);
82+
GenericData.Record userResult = (GenericData.Record) kryo.fromBytes(userBytes);
83+
assertEquals(userResult.get("name").toString(),"Jeff");
84+
assertEquals(userResult.get("ID"),1);
85+
assertEquals(user.toString(), userResult.toString());
86+
87+
}
88+
89+
private <T> ClassTag<T> getClassTag(Class<T> klass) {
90+
return scala.reflect.ClassTag$.MODULE$.apply(klass);
91+
}
92+
}

‎chill-avro/src/test/scala/com/twitter/chill/avro/AvroSerializerSpec.scala

+4-2
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,15 @@ import org.apache.avro.generic.GenericRecordBuilder
2121
import org.apache.avro.SchemaBuilder
2222
import org.apache.avro.generic.GenericData.Record
2323

24+
import scala.reflect.ClassTag
25+
2426
/**
2527
* @author Mansur Ashraf
2628
* @since 2/9/14.
2729
*/
28-
object AvroSerializerSpec extends WordSpec with Matchers {
30+
class AvroSerializerSpec extends WordSpec with Matchers {
2931

30-
def getKryo[T: Manifest](k: KSerializer[T]) = {
32+
def getKryo[T: ClassTag](k: KSerializer[T]) = {
3133
val inst = {
3234
() => (new ScalaKryoInstantiator).newKryo.forClass(k)
3335
}

‎chill-hadoop/src/main/java/com/twitter/chill/hadoop/KryoSerialization.java

+13-10
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ public class KryoSerialization extends Configured implements Serialization<Objec
5050
* It will first call this, then setConf.
5151
*/
5252
public KryoSerialization() {
53-
this(new Configuration());
53+
super();
5454
}
5555

5656
/**
@@ -65,15 +65,18 @@ public KryoSerialization( Configuration conf ) {
6565

6666
@Override
6767
public void setConf(Configuration conf) {
68-
try {
69-
KryoInstantiator kryoInst = new ConfiguredInstantiator(new HadoopConfig(conf));
70-
testKryo = kryoInst.newKryo();
71-
kryoPool = KryoPool.withByteArrayOutputStream(MAX_CACHED_KRYO, kryoInst);
72-
}
73-
catch(ConfigurationException cx) {
74-
// This interface can't throw
75-
throw new RuntimeException(cx);
76-
}
68+
// null check is to handle when calling the defaul constructor, in Configured, it calls super which calls setConf with a null Configuration
69+
if (conf != null) {
70+
try {
71+
KryoInstantiator kryoInst = new ConfiguredInstantiator(new HadoopConfig(conf));
72+
testKryo = kryoInst.newKryo();
73+
kryoPool = KryoPool.withByteArrayOutputStream(MAX_CACHED_KRYO, kryoInst);
74+
}
75+
catch(ConfigurationException cx) {
76+
// This interface can't throw
77+
throw new RuntimeException(cx);
78+
}
79+
}
7780
}
7881

7982
/**

‎chill-java/src/main/java/com/twitter/chill/config/Config.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ public String getOrElse(String key, String def) {
3131
}
3232

3333
public boolean contains(String key) {
34-
return get(key) == null;
34+
return get(key) != null;
3535
}
3636
public Boolean getBoolean(String key) {
3737
String bval = get(key);

‎project/Build.scala

+5-6
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ object ChillBuild extends Build {
2020

2121
val sharedSettings = Project.defaultSettings ++ mimaDefaultSettings ++ scalariformSettings ++ Seq(
2222

23-
version := "0.5.0",
23+
version := "0.5.1",
2424
organization := "com.twitter",
2525
scalaVersion := "2.10.4",
2626
crossScalaVersions := Seq("2.10.4", "2.11.2"),
@@ -125,8 +125,8 @@ object ChillBuild extends Build {
125125
Some(subProj)
126126
.filterNot(unreleasedModules.contains(_))
127127
.map { s =>
128-
val suffix = if (javaOnly.contains(s)) "" else "_2.9.3"
129-
"com.twitter" % ("chill-" + s + suffix) % "0.3.6"
128+
val suffix = if (javaOnly.contains(s)) "" else "_2.10"
129+
"com.twitter" % ("chill-" + s + suffix) % "0.5.1"
130130
}
131131

132132
def module(name: String) = {
@@ -225,10 +225,9 @@ object ChillBuild extends Build {
225225
).dependsOn(chillJava)
226226

227227
lazy val chillAvro = module("avro").settings(
228-
crossPaths := false,
229-
autoScalaLibrary := false,
230228
libraryDependencies ++= Seq(
231-
"com.twitter" %% "bijection-avro" % "0.7.0"
229+
"com.twitter" %% "bijection-avro" % "0.7.0",
230+
"junit" % "junit" % "4.5" % "test"
232231
)
233232
).dependsOn(chill,chillJava, chillBijection)
234233

0 commit comments

Comments
 (0)