diff --git a/build.gradle b/build.gradle index 2d3da4c2..212c4244 100644 --- a/build.gradle +++ b/build.gradle @@ -2,7 +2,7 @@ import java.util.zip.ZipFile apply plugin: 'java' apply plugin: 'maven' -version="2.47" +version="2.50" targetCompatibility = 1.7 sourceCompatibility = 1.7 @@ -18,9 +18,9 @@ repositories { dependencies { compile fileTree("src/test/lib") - compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.5.3' - compile group: 'org.javassist', name: 'javassist', version: '3.19.0-GA' - compile 'org.objenesis:objenesis:2.4' + compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.8.8' + compile group: 'org.javassist', name: 'javassist', version: '3.21.0-GA' + compile 'org.objenesis:objenesis:2.5.1' } task sourcesJar(type: Jar, dependsOn:classes) { @@ -39,9 +39,9 @@ task testJar(type: Jar, dependsOn:classes) { task allJar(type: Jar, dependsOn:jar) { archiveName = 'fst-'+version+"-onejar.jar" - from zipTree('lib/objenesis-2.4.jar').matching { true } - from zipTree('lib/jackson-core-2.5.3.jar').matching { true } - from zipTree('lib/javassist-3.19.0-GA.jar').matching { true } + from zipTree('lib/objenesis-2.5.1.jar').matching { true } + from zipTree('lib/jackson-core-2.8.8.jar').matching { true } + from zipTree('lib/javassist-3.21.0-GA.jar').matching { true } from zipTree('build/libs/fst-'+version+'.jar').matching { true } } diff --git a/pom.xml b/pom.xml index e52fe0d0..ff5d8c96 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ 4.0.0 de.ruedigermoeller fst - 2.49 + 2.50 bundle A fast java serialization drop in-replacement and some serialization based utils such as Structs and OffHeap Memory. diff --git a/src/main/java/org/nustaq/kson/Kson.java b/src/main/java/org/nustaq/kson/Kson.java index fcd2391c..74284c58 100644 --- a/src/main/java/org/nustaq/kson/Kson.java +++ b/src/main/java/org/nustaq/kson/Kson.java @@ -22,6 +22,8 @@ import java.lang.reflect.Field; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.Map; import java.util.Scanner; /** @@ -191,4 +193,47 @@ public String writePlainJSonObject(Object o) throws Exception { public KsonTypeMapper getMapper() { return mapper; } + + static class Test implements Serializable { + Map> map; + Map[] arr; + } + + public static void main(String[] args) throws Exception { + Test test = new Test(); + test.map = new HashMap<>(); + HashMap mp = new HashMap(); + mp.put("x","y"); + mp.put("xx","xy"); + mp.put("xxx","xxyx"); + test.map.put("pok",mp); + test.arr = new Map[] { mp, mp }; + System.out.println(new Kson().writeObject(test)); + Object res = new Kson().map(Test.class).readObject( +"Test {\n" + + " map:\n" + + " {\n" + + " pok : \n" + + " {\n" + + " xx : xy\n" + + " x : y\n" + + " xxx : xxyx\n" + + " }\n" + + " }\n" + + " arr:\n" + + " [ \n" + + " {\n" + + " xx : xy\n" + + " x : y\n" + + " xxx : xxyx\n" + + " }\n" + + " {\n" + + " xx : xy\n" + + " x : y\n" + + " xxx : xxyx\n" + + " }\n" + + " ]\n" + + "}" ); + int debug = 12; + } } diff --git a/src/main/java/org/nustaq/offheap/BinaryQueue.java b/src/main/java/org/nustaq/offheap/BinaryQueue.java index 66763175..7795aa9d 100644 --- a/src/main/java/org/nustaq/offheap/BinaryQueue.java +++ b/src/main/java/org/nustaq/offheap/BinaryQueue.java @@ -13,7 +13,7 @@ * read* methods throw excpetions in case the q contains not enough bytes (need to use available() to * ensure data is present). * - * add methods add to the queue. If the adding outperforms reading/polling, the queue is growed automatically. + * add methods add to the queue. If the adding outperforms reading/polling, the queue is grown automatically. * */ public class BinaryQueue { diff --git a/src/main/java/org/nustaq/offheap/FSTAsciiStringOffheapMap.java b/src/main/java/org/nustaq/offheap/FSTAsciiStringOffheapMap.java index 2dd55d5d..84045cc4 100644 --- a/src/main/java/org/nustaq/offheap/FSTAsciiStringOffheapMap.java +++ b/src/main/java/org/nustaq/offheap/FSTAsciiStringOffheapMap.java @@ -82,8 +82,18 @@ public FSTAsciiStringOffheapMap(String file, int keylen, long size, int numelem) } public ByteSource encodeKey(String key) { - if ( key.length() > tmpKey.length() ) - throw new RuntimeException("key too long: '"+key+"' maxlen:"+tmpKey.length()); + if ( key.length() > tmpKey.length() ) { + int length = (int) tmpKey.length(); + StringBuilder buf = new StringBuilder(length); + for ( int i = 0; i < length; i++ ) { + char c = key.charAt(i); + if ( i+length < key.length() ) { + c = (char) ((c + key.charAt(i+length))/2); + } + buf.append(c); + } + key = buf.toString(); + } tmpKey.setString(key); return tmpKey; } diff --git a/src/main/java/org/nustaq/offheap/FSTBinaryOffheapMap.java b/src/main/java/org/nustaq/offheap/FSTBinaryOffheapMap.java index fcbe6172..0c31dbb3 100644 --- a/src/main/java/org/nustaq/offheap/FSTBinaryOffheapMap.java +++ b/src/main/java/org/nustaq/offheap/FSTBinaryOffheapMap.java @@ -269,14 +269,22 @@ protected long addEntry(ByteSource key, ByteSource value) { return res; } + private void resizeStore(long required) { + resizeStore(required,GB); + } + /** + * PRIVILEGED method. You gotta know what your doing here .. + * * currently a very expensive operation .. frees everything, resize file and remap. * Remapping involves rebuild of index. * @param required */ - private void resizeStore(long required) { + public void resizeStore(long required, long maxgrowbytes) { if ( mappedFile == null ) throw new RuntimeException("store is full. Required: "+required); + if ( required <= memory.length() ) + return; mutationCount++; System.out.println("resizing underlying "+mappedFile+" to "+required+" numElem:"+numElem); long tim = System.currentTimeMillis(); @@ -286,7 +294,7 @@ private void resizeStore(long required) { File mf = new File(mappedFile); FileOutputStream f = new FileOutputStream(mf,true); long len = mf.length(); - required = required + Math.min(required,MB*512); + required = required + Math.min(required,maxgrowbytes); byte[] toWrite = new byte[1000]; long max = (required - len)/1000; for ( long i = 0; i < max+2; i++ ) { diff --git a/src/main/java/org/nustaq/serialization/FSTClazzInfoRegistry.java b/src/main/java/org/nustaq/serialization/FSTClazzInfoRegistry.java index 022d168e..4ac8e8b4 100644 --- a/src/main/java/org/nustaq/serialization/FSTClazzInfoRegistry.java +++ b/src/main/java/org/nustaq/serialization/FSTClazzInfoRegistry.java @@ -84,7 +84,14 @@ static void addAllReferencedClasses(Class cl, HashSet names, HashSetrethrow(e); } return null; diff --git a/src/main/java/org/nustaq/serialization/FSTEncoder.java b/src/main/java/org/nustaq/serialization/FSTEncoder.java index d10b62bd..2d538277 100644 --- a/src/main/java/org/nustaq/serialization/FSTEncoder.java +++ b/src/main/java/org/nustaq/serialization/FSTEncoder.java @@ -85,7 +85,8 @@ public interface FSTEncoder { // write a meta byte item. return true if encoder wrote full object (e.g. literal, primitive) boolean writeTag(byte tag, Object info, long somValue, Object toWrite, FSTObjectOutput oout) throws IOException; - void writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo); + // return true, if this already wrote everything + boolean writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo, Object value); void externalEnd(FSTClazzInfo clz); // demarkls the end of an externalizable or classes with serializer registered diff --git a/src/main/java/org/nustaq/serialization/FSTObjectOutput.java b/src/main/java/org/nustaq/serialization/FSTObjectOutput.java index d7e94e10..d95cc717 100644 --- a/src/main/java/org/nustaq/serialization/FSTObjectOutput.java +++ b/src/main/java/org/nustaq/serialization/FSTObjectOutput.java @@ -271,7 +271,6 @@ public void writeUTF(String s) throws IOException { ///////////////////////////////////////////////////// public void writeObject(Object obj, Class... possibles) throws IOException { - curDepth++; if ( isCrossPlatform ) { writeObjectInternal(obj, null); // not supported cross platform return; @@ -312,9 +311,6 @@ protected FSTClazzInfo.FSTFieldInfo getCachedFI( Class... possibles ) { * @throws IOException */ public FSTClazzInfo writeObjectInternal(Object obj, FSTClazzInfo ci, Class... possibles) throws IOException { - if ( curDepth == 0 ) { - throw new RuntimeException("not intended to be called from external application. Use public writeObject instead"); - } FSTClazzInfo.FSTFieldInfo info = getCachedFI(possibles); curDepth++; FSTClazzInfo fstClazzInfo = writeObjectWithContext(info, obj, ci); @@ -600,7 +596,9 @@ protected void writeObjectFields(Object toWrite, FSTClazzInfo serializationInfo, writeObjectFields(toWrite, serializationInfo, fieldInfo, i, subInfo.getVersion()); return; } - getCodec().writeAttributeName(subInfo); + if ( getCodec().writeAttributeName(subInfo, toWrite) ) { + continue; + } if ( subInfo.isPrimitive() ) { // speed safe int integralType = subInfo.getIntegralType(); diff --git a/src/main/java/org/nustaq/serialization/FSTSerializerRegistry.java b/src/main/java/org/nustaq/serialization/FSTSerializerRegistry.java index 0ad545ba..36f50321 100644 --- a/src/main/java/org/nustaq/serialization/FSTSerializerRegistry.java +++ b/src/main/java/org/nustaq/serialization/FSTSerializerRegistry.java @@ -28,6 +28,7 @@ * */ public class FSTSerializerRegistry { + private FSTSerializerRegistryDelegate delegate; public static FSTObjectSerializer NULL = new NULLSerializer(); diff --git a/src/main/java/org/nustaq/serialization/coders/FSTBytezEncoder.java b/src/main/java/org/nustaq/serialization/coders/FSTBytezEncoder.java index 41639df1..a0e5064d 100644 --- a/src/main/java/org/nustaq/serialization/coders/FSTBytezEncoder.java +++ b/src/main/java/org/nustaq/serialization/coders/FSTBytezEncoder.java @@ -179,8 +179,8 @@ public void writeFShort(short c) throws IOException { writePlainShort(c); } - public void writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo) { - + public boolean writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo, Object value) { + return false; } public boolean writeTag(byte tag, Object info, long somValue, Object toWrite, FSTObjectOutput oout) throws IOException { diff --git a/src/main/java/org/nustaq/serialization/coders/FSTJsonDecoder.java b/src/main/java/org/nustaq/serialization/coders/FSTJsonDecoder.java index ff3d547f..70f4da06 100644 --- a/src/main/java/org/nustaq/serialization/coders/FSTJsonDecoder.java +++ b/src/main/java/org/nustaq/serialization/coders/FSTJsonDecoder.java @@ -7,6 +7,7 @@ import java.io.IOException; import java.io.InputStream; +import java.io.UnsupportedEncodingException; import java.lang.reflect.Array; import java.util.ArrayList; import java.util.HashMap; @@ -627,6 +628,13 @@ public void readObjectEnd() { @Override public Object coerceElement(Class arrType, Object value) { + if ( arrType == byte[].class && value instanceof String ) { + try { + return ((String) value).getBytes("UTF-8"); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + } if ( value instanceof Number ) { Number n = (Number) value; if ( arrType == Byte.class ) { diff --git a/src/main/java/org/nustaq/serialization/coders/FSTJsonEncoder.java b/src/main/java/org/nustaq/serialization/coders/FSTJsonEncoder.java index 32a62500..f54fd7bf 100644 --- a/src/main/java/org/nustaq/serialization/coders/FSTJsonEncoder.java +++ b/src/main/java/org/nustaq/serialization/coders/FSTJsonEncoder.java @@ -13,6 +13,8 @@ import java.lang.reflect.Array; import java.util.Map; +import static org.nustaq.serialization.FSTObjectOutput.STRING; + /** * Created by ruedi on 20/05/15. * @@ -386,7 +388,7 @@ protected String classToString(Class clz) { } @Override - public void writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo) { + public boolean writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo, Object outerObjectToWrite) { try { SerializedString bufferedName = (SerializedString) subInfo.getBufferedName(); if ( bufferedName == null ) { @@ -399,9 +401,22 @@ public void writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo) { else { gen.writeFieldName(bufferedName); } + if ( subInfo.getField().isAnnotationPresent(JSONAsString.class) ) { // fixme: optimize + try { + Object objectValue = subInfo.getObjectValue(outerObjectToWrite); + if ( objectValue instanceof byte[] ) { + String stringVal = new String((byte[]) objectValue, "UTF-8"); + writeStringUTF(stringVal); + return true; + } + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + } } catch (IOException e) { FSTUtil.rethrow(e); } + return false; } @Override diff --git a/src/main/java/org/nustaq/serialization/coders/FSTMinBinEncoder.java b/src/main/java/org/nustaq/serialization/coders/FSTMinBinEncoder.java index 9482a363..043e838c 100644 --- a/src/main/java/org/nustaq/serialization/coders/FSTMinBinEncoder.java +++ b/src/main/java/org/nustaq/serialization/coders/FSTMinBinEncoder.java @@ -194,7 +194,7 @@ public void writeClass(FSTClazzInfo clInf) { } @Override - public void writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo) { + public boolean writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo, Object value) { byte[] bufferedName = (byte[]) subInfo.getBufferedName(); if ( bufferedName != null ) { out.writeRaw(bufferedName,0,bufferedName.length); @@ -209,6 +209,7 @@ public void writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo) { System.arraycopy(out.getBytez(),pos,bufferedName,0,len); subInfo.setBufferedName(bufferedName); } + return false; } @Override diff --git a/src/main/java/org/nustaq/serialization/coders/FSTStreamEncoder.java b/src/main/java/org/nustaq/serialization/coders/FSTStreamEncoder.java index 1d2d2401..f1f756df 100644 --- a/src/main/java/org/nustaq/serialization/coders/FSTStreamEncoder.java +++ b/src/main/java/org/nustaq/serialization/coders/FSTStreamEncoder.java @@ -343,8 +343,8 @@ public void writeFShort(short c) throws IOException { } } - public void writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo) { - + public boolean writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo, Object value) { + return false; } public boolean writeTag(byte tag, Object info, long somValue, Object toWrite, FSTObjectOutput oout) throws IOException { diff --git a/src/main/java/org/nustaq/serialization/coders/JSONAsString.java b/src/main/java/org/nustaq/serialization/coders/JSONAsString.java new file mode 100644 index 00000000..2b701744 --- /dev/null +++ b/src/main/java/org/nustaq/serialization/coders/JSONAsString.java @@ -0,0 +1,19 @@ +package org.nustaq.serialization.coders; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Created by ruedi on 21.03.17. + * + * advice json serializer to convert annotated field to a string (e.g. byte or char arrays) + * + * !!!!!!!! supports byte[] only currently !!!!!!!!!!!!!!! + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.FIELD) + +public @interface JSONAsString { +}