diff --git a/pom.xml b/pom.xml index 84b1cce6..efbab67d 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ 4.0.0 de.ruedigermoeller fst - 2.51 + 2.52 bundle A fast java serialization drop in-replacement and some serialization based utils such as Structs and OffHeap Memory. diff --git a/src/main/java/org/nustaq/serialization/FSTConfiguration.java b/src/main/java/org/nustaq/serialization/FSTConfiguration.java index fa8a5422..42289f4f 100644 --- a/src/main/java/org/nustaq/serialization/FSTConfiguration.java +++ b/src/main/java/org/nustaq/serialization/FSTConfiguration.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.core.ObjectCodec; import com.fasterxml.jackson.core.SerializableString; import com.fasterxml.jackson.core.io.IOContext; +import com.fasterxml.jackson.core.json.JsonWriteContext; import com.fasterxml.jackson.core.json.UTF8JsonGenerator; import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; import org.nustaq.offheap.bytez.onheap.HeapBytez; @@ -36,12 +37,14 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; import java.util.*; import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; + /** * Created with IntelliJ IDEA. * User: ruedi @@ -257,7 +260,10 @@ public static FSTConfiguration createJsonNoRefConfiguration() { /** * create a json conf with given attributes. Note that shared refs = true for jason might be not as stable as for binary encodings * as fst relies on stream positions to identify objects within a given input, so any inbetween formatting will break proper reference - * resolution + * resolution. + * + * WARNING: use of sharedrefs = true is Deprecated as its flakey + * * @param prettyPrint * @param shareReferences * @return @@ -269,6 +275,10 @@ public static FSTConfiguration createJsonConfiguration(boolean prettyPrint, bool return createJsonConfiguration(prettyPrint,shareReferences,null); } + public static FSTConfiguration createJsonConfiguration() { + return createJsonConfiguration(false,false); + } + protected static FSTConfiguration createJsonConfiguration(boolean prettyPrint, boolean shareReferences, ConcurrentHashMap shared ) { if ( prettyPrint && shareReferences ) { throw new RuntimeException("cannot use prettyPrint with shared refs to 'true'. Set shareRefs to false."); @@ -467,7 +477,7 @@ protected static FSTConfiguration initDefaultFstConfigurationInternal(FSTConfigu // for most cases don't register for subclasses as in many cases we'd like to fallback to JDK implementation // (e.g. TreeMap) in order to guarantee complete serialization reg.putSerializer(ArrayList.class, new FSTArrayListSerializer(), false); - reg.putSerializer(Vector.class, new FSTCollectionSerializer(), false); + reg.putSerializer(Vector.class, new FSTCollectionSerializer(), true); reg.putSerializer(LinkedList.class, new FSTCollectionSerializer(), false); // subclass should register manually reg.putSerializer(HashSet.class, new FSTCollectionSerializer(), false); // subclass should register manually reg.putSerializer(HashMap.class, new FSTMapSerializer(), false); // subclass should register manually @@ -923,6 +933,8 @@ public FSTObjectInput getObjectInputCopyFrom( byte arr[],int off, int len ) { protected FSTObjectInput getIn() { FSTObjectInput fstObjectInput = (FSTObjectInput) streamCoderFactory.getInput().get(); + if ( fstObjectInput != null && fstObjectInput.isClosed() ) + fstObjectInput = null; if ( fstObjectInput == null ) { streamCoderFactory.getInput().set(new FSTObjectInput(this)); return getIn(); @@ -1191,13 +1203,8 @@ public String asJsonString(Object o) { if ( getCoderSpecific() instanceof JsonFactory == false ) { return "can be called on JsonConfiguration only"; } else { - try { - return new String(asByteArray(o),"UTF-8"); - } catch (UnsupportedEncodingException e) { - FSTUtil.rethrow(e); - } + return new String(asByteArray(o), StandardCharsets.UTF_8); } - return null; } /** diff --git a/src/main/java/org/nustaq/serialization/FSTObjectInput.java b/src/main/java/org/nustaq/serialization/FSTObjectInput.java index 36c4950b..0f177921 100644 --- a/src/main/java/org/nustaq/serialization/FSTObjectInput.java +++ b/src/main/java/org/nustaq/serialization/FSTObjectInput.java @@ -147,6 +147,10 @@ protected void setCodec(FSTDecoder codec) { this.codec = codec; } + public boolean isClosed() { + return closed; + } + protected static class CallbackEntry { ObjectInputValidation cb; int prio; diff --git a/src/main/java/org/nustaq/serialization/FSTObjectOutput.java b/src/main/java/org/nustaq/serialization/FSTObjectOutput.java index d5852d13..66bfa806 100644 --- a/src/main/java/org/nustaq/serialization/FSTObjectOutput.java +++ b/src/main/java/org/nustaq/serialization/FSTObjectOutput.java @@ -451,10 +451,21 @@ protected FSTClazzInfo writeObjectWithContext(FSTClazzInfo.FSTFieldInfo referenc return originalInfo; } } + if (! writeObjectHeader(serializationInfo, referencee, toWrite) ) { // skip in case codec can write object as primitive - defaultWriteObject(toWrite, serializationInfo); - if ( serializationInfo.isExternalizable() ) + ser = serializationInfo.getSer(); + if ( ser == null ) { + defaultWriteObject(toWrite, serializationInfo); + if ( serializationInfo.isExternalizable() ) + getCodec().externalEnd(serializationInfo); + } else { + // handle edge case: there is a serializer registered for replaced class + // copied from below :( + int pos = getCodec().getWritten(); + // write object depending on type (custom, externalizable, serializable/java, default) + ser.writeObject(this, toWrite, serializationInfo, referencee, pos); getCodec().externalEnd(serializationInfo); + } } return originalInfo; } else { // object has custom serializer diff --git a/src/main/java/org/nustaq/serialization/coders/FSTJsonEncoder.java b/src/main/java/org/nustaq/serialization/coders/FSTJsonEncoder.java index f54fd7bf..eb37318f 100644 --- a/src/main/java/org/nustaq/serialization/coders/FSTJsonEncoder.java +++ b/src/main/java/org/nustaq/serialization/coders/FSTJsonEncoder.java @@ -3,7 +3,9 @@ import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonStreamContext; +import com.fasterxml.jackson.core.ObjectCodec; import com.fasterxml.jackson.core.io.SerializedString; +import com.fasterxml.jackson.core.json.JsonWriteContext; import org.nustaq.serialization.*; import org.nustaq.serialization.util.FSTOutputStream; import org.nustaq.serialization.util.FSTUtil; @@ -168,6 +170,13 @@ public void close() throws IOException { @Override public void reset(byte[] outbytes) { + if ( gen != null ) { + try { + createGenerator(); + } catch (Exception e) { + FSTUtil.rethrow(e); + } + } if (outbytes==null) { out.reset(); } else { diff --git a/src/test/json/StringEnc.java b/src/test/json/StringEnc.java index 8844286e..5b86c9a0 100644 --- a/src/test/json/StringEnc.java +++ b/src/test/json/StringEnc.java @@ -31,4 +31,20 @@ public void testStringenc() throws UnsupportedEncodingException { Assert.assertTrue(x.equals("ÄÖasdß")); } + static class PJ implements Serializable { + int a = 10; + } + + @Test + public void testLeadingSpaceBug() { + FSTConfiguration fst = FSTConfiguration.createJsonConfiguration(); + PJ pojo = new PJ(); + String x = fst.asJsonString(pojo); + System.out.println(x); + String x1 = fst.asJsonString(pojo); + System.out.println(x1); + Assert.assertTrue(x.equals(x1)); + + } + } diff --git a/src/test/offheap/StringOffHeapTest.java b/src/test/offheap/StringOffHeapTest.java index 1b62f3eb..55112805 100644 --- a/src/test/offheap/StringOffHeapTest.java +++ b/src/test/offheap/StringOffHeapTest.java @@ -148,7 +148,7 @@ public void hardcore() throws Exception { FSTCoder coder = createCoder(); coder.getConf().registerClass(TestRec.class); - int count = 5; + int count = 2; // while (true) while (count-- > 0) { diff --git a/src/test/ser/RawMemTest.java b/src/test/ser/RawMemTest.java index f39ab6b5..6121c5fc 100644 --- a/src/test/ser/RawMemTest.java +++ b/src/test/ser/RawMemTest.java @@ -49,18 +49,12 @@ public void test() { System.out.println("binary"); deser = smallBench(conf, original, deser); deser = smallBench(conf, original, deser); - deser = smallBench(conf, original, deser); - deser = smallBench(conf, original, deser); - deser = smallBench(conf, original, deser); assertTrue(DeepEquals.deepEquals(original, deser)); System.out.println("default"); conf = FSTConfiguration.createDefaultConfiguration(); deser = smallBench(conf, original, deser); deser = smallBench(conf, original, deser); - deser = smallBench(conf, original, deser); - deser = smallBench(conf, original, deser); - deser = smallBench(conf, original, deser); assertTrue(DeepEquals.deepEquals(original, deser)); System.out.println("Default LEN:"+FSTConfiguration.createDefaultConfiguration().asByteArray(original).length); @@ -99,8 +93,6 @@ public void testOffHeapCoder0( boolean shared ) throws Exception { MallocBytezAllocator alloc = new MallocBytezAllocator(); MallocBytez bytez = (MallocBytez) alloc.alloc(1000 * 1000); - ohbench(original, coder, bytez); - ohbench(original, coder, bytez); ohbench(original, coder, bytez); ohbench(original, coder, bytez); Object deser = ohbench(original, coder, bytez); @@ -109,8 +101,6 @@ public void testOffHeapCoder0( boolean shared ) throws Exception { System.out.println("-----"); ohbench(smallClazz, coder, bytez); ohbench(smallClazz, coder, bytez); - ohbench(smallClazz, coder, bytez); - ohbench(smallClazz, coder, bytez); deser = ohbench(smallClazz, coder, bytez); assertTrue(DeepEquals.deepEquals(smallClazz, deser)); @@ -158,8 +148,6 @@ public void testOnHeapCoder0(boolean shared) throws Exception { Object deser = coder.toObject(arr, 0, (int) arr.length); assertTrue(DeepEquals.deepEquals(deser,original)); - onhbench(original, coder, arr, 0); - onhbench(original, coder, arr, 0); onhbench(original, coder, arr, 0); onhbench(original, coder, arr, 0); deser = onhbench(original, coder, arr, 0); @@ -168,9 +156,6 @@ public void testOnHeapCoder0(boolean shared) throws Exception { System.out.println("-----"); deser = onhbench(smallClazz, coder, arr, 0); deser = onhbench(smallClazz, coder, arr, 0); - deser = onhbench(smallClazz, coder, arr, 0); - deser = onhbench(smallClazz, coder, arr, 0); - deser = onhbench(smallClazz, coder, arr, 0); assertTrue(DeepEquals.deepEquals(smallClazz, deser)); boolean lenEx = false; diff --git a/src/test/ser/serializers/Github204.java b/src/test/ser/serializers/Github204.java new file mode 100644 index 00000000..c39e2bb1 --- /dev/null +++ b/src/test/ser/serializers/Github204.java @@ -0,0 +1,142 @@ +package ser.serializers; + +import org.junit.Test; +import org.nustaq.serialization.FSTConfiguration; +import org.nustaq.serialization.FSTObjectInput; +import org.nustaq.serialization.FSTObjectOutput; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectStreamException; +import java.io.Serializable; +import java.util.*; + +import static org.junit.Assert.assertEquals; + +/** + * Created by ruedi on 31.07.17. + */ +public class Github204 { + + @Test + public void testMapWriteReplace() throws Exception { + Map map = new MyMap<>(); + map.put("value", 1234.5); + + Map map1 = new MyMap<>(); + map1.put("othervalue", 5678.9); + + ByteArrayOutputStream buf = new ByteArrayOutputStream(); + FSTObjectOutput out = new FSTObjectOutput(buf, getTestConfiguration()); + out.writeObject(map); + out.writeObject(map1); + out.writeObject("Ensure Stream is not corrupted"); + out.close(); + + FSTObjectInput in = getTestConfiguration().getObjectInput(new ByteArrayInputStream(buf.toByteArray())); + Map res = (Map) in.readObject(); + Map res1 = (Map) in.readObject(); + in.close(); + + assertEquals(LinkedHashMap.class, res.getClass()); + assertEquals(1, res.size()); + assertEquals(1234.5, res.get("value")); + assertEquals(LinkedHashMap.class, res1.getClass()); + assertEquals(1, res1.size()); + assertEquals(5678.9, res1.get("othervalue")); + } + + private FSTConfiguration getTestConfiguration() { + return FSTConfiguration.createDefaultConfiguration(); + } + + public static class MyMap implements Map, Serializable { + private final Map data; + + public MyMap() { + this(new HashMap()); + } + + + public MyMap(Map data) { + this.data = data; + } + + @Override + public int size() { + return data.size(); + } + + @Override + public boolean isEmpty() { + return data.isEmpty(); + } + + @Override + public boolean containsKey(Object key) { + return data.containsKey(key); + } + + @Override + public boolean containsValue(Object value) { + return data.containsValue(value); + } + + @Override + public V get(Object key) { + return data.get(key); + } + + @Override + public V put(K key, V value) { + return data.put(key, value); + } + + @Override + public V remove(Object key) { + return data.remove(key); + } + + @Override + public void putAll(Map m) { + data.putAll(m); + } + + @Override + public void clear() { + data.clear(); + } + + @Override + public Set keySet() { + return data.keySet(); + } + + @Override + public Collection values() { + return data.values(); + } + + @Override + public Set> entrySet() { + return data.entrySet(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + MyMap myMap = (MyMap) o; + return Objects.equals(data, myMap.data); + } + + @Override + public int hashCode() { + return Objects.hash(data); + } + + private Object writeReplace() throws ObjectStreamException { + return new LinkedHashMap(this); + } + } +}