v2.50

RuedigerMoeller · Apr 10, 2017 · 91d1e4c · 91d1e4c
1 parent f0f1825
commit 91d1e4c
Show file tree

Hide file tree

Showing 17 changed files with 151 additions and 27 deletions.
diff --git a/build.gradle b/build.gradle
@@ -2,7 +2,7 @@ import java.util.zip.ZipFile
 
 apply plugin: 'java'
 apply plugin: 'maven'
-version="2.47"
+version="2.50"
 
 targetCompatibility = 1.7
 sourceCompatibility = 1.7
@@ -18,9 +18,9 @@ repositories {
 
 dependencies {
     compile fileTree("src/test/lib")
-    compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.5.3'
-    compile group: 'org.javassist', name: 'javassist', version: '3.19.0-GA'
-    compile 'org.objenesis:objenesis:2.4'
+    compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.8.8'
+    compile group: 'org.javassist', name: 'javassist', version: '3.21.0-GA'
+    compile 'org.objenesis:objenesis:2.5.1'
 }
 
 task sourcesJar(type: Jar, dependsOn:classes) {
@@ -39,9 +39,9 @@ task testJar(type: Jar, dependsOn:classes) {
 
 task allJar(type: Jar, dependsOn:jar) {
     archiveName = 'fst-'+version+"-onejar.jar"
-    from zipTree('lib/objenesis-2.4.jar').matching { true }
-    from zipTree('lib/jackson-core-2.5.3.jar').matching { true }
-    from zipTree('lib/javassist-3.19.0-GA.jar').matching { true }
+    from zipTree('lib/objenesis-2.5.1.jar').matching { true }
+    from zipTree('lib/jackson-core-2.8.8.jar').matching { true }
+    from zipTree('lib/javassist-3.21.0-GA.jar').matching { true }
     from zipTree('build/libs/fst-'+version+'.jar').matching { true }
 }
 

diff --git a/pom.xml b/pom.xml
@@ -10,7 +10,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>de.ruedigermoeller</groupId>
     <artifactId>fst</artifactId>
-    <version>2.49</version>
+    <version>2.50</version>
     <packaging>bundle</packaging>
 
     <description>A fast java serialization drop in-replacement and some serialization based utils such as Structs and OffHeap Memory.</description>

diff --git a/src/main/java/org/nustaq/kson/Kson.java b/src/main/java/org/nustaq/kson/Kson.java
@@ -22,6 +22,8 @@
 import java.lang.reflect.Field;
 import java.lang.reflect.ParameterizedType;
 import java.lang.reflect.Type;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Scanner;
 
 /**
@@ -191,4 +193,47 @@ public String writePlainJSonObject(Object o) throws Exception {
     public KsonTypeMapper getMapper() {
         return mapper;
     }
+
+    static class Test implements Serializable {
+        Map<String,Map<String,String>> map;
+        Map<String,String>[] arr;
+    }
+
+    public static void main(String[] args) throws Exception {
+        Test test = new Test();
+        test.map = new HashMap<>();
+        HashMap mp = new HashMap();
+        mp.put("x","y");
+        mp.put("xx","xy");
+        mp.put("xxx","xxyx");
+        test.map.put("pok",mp);
+        test.arr = new Map[] { mp, mp };
+        System.out.println(new Kson().writeObject(test));
+        Object res = new Kson().map(Test.class).readObject(
+"Test {\n" +
+    "  map:\n" +
+    "    {\n" +
+    "      pok : \n" +
+    "      {\n" +
+    "        xx :  xy\n" +
+    "        x :  y\n" +
+    "        xxx :  xxyx\n" +
+    "      }\n" +
+    "    }\n" +
+    "  arr:\n" +
+    "    [ \n" +
+    "      {\n" +
+    "        xx :  xy\n" +
+    "        x :  y\n" +
+    "        xxx :  xxyx\n" +
+    "      }\n" +
+    "      {\n" +
+    "        xx :  xy\n" +
+    "        x :  y\n" +
+    "        xxx :  xxyx\n" +
+    "      }\n" +
+    "    ]\n" +
+    "}"            );
+        int debug = 12;
+    }
 }
diff --git a/src/main/java/org/nustaq/offheap/BinaryQueue.java b/src/main/java/org/nustaq/offheap/BinaryQueue.java
@@ -13,7 +13,7 @@
  * read* methods throw excpetions in case the q contains not enough bytes (need to use available() to
  * ensure data is present).
  *
- * add methods add to the queue. If the adding outperforms reading/polling, the queue is growed automatically.
+ * add methods add to the queue. If the adding outperforms reading/polling, the queue is grown automatically.
  *
  */
 public class BinaryQueue {

diff --git a/src/main/java/org/nustaq/offheap/FSTAsciiStringOffheapMap.java b/src/main/java/org/nustaq/offheap/FSTAsciiStringOffheapMap.java
@@ -82,8 +82,18 @@ public FSTAsciiStringOffheapMap(String file, int keylen, long size, int numelem)
     }
 
     public ByteSource encodeKey(String key) {
-        if ( key.length() > tmpKey.length() )
-            throw new RuntimeException("key too long: '"+key+"' maxlen:"+tmpKey.length());
+        if ( key.length() > tmpKey.length() ) {
+            int length = (int) tmpKey.length();
+            StringBuilder buf = new StringBuilder(length);
+            for ( int i = 0; i < length; i++ ) {
+                char c = key.charAt(i);
+                if ( i+length < key.length() ) {
+                    c = (char) ((c + key.charAt(i+length))/2);
+                }
+                buf.append(c);
+            }
+            key = buf.toString();
+        }
         tmpKey.setString(key);
         return tmpKey;
     }

diff --git a/src/main/java/org/nustaq/offheap/FSTBinaryOffheapMap.java b/src/main/java/org/nustaq/offheap/FSTBinaryOffheapMap.java
@@ -269,14 +269,22 @@ protected long addEntry(ByteSource key, ByteSource value) {
         return res;
     }
 
+    private void resizeStore(long required) {
+        resizeStore(required,GB);
+    }
+
     /**
+     * PRIVILEGED method. You gotta know what your doing here ..
+     *
      * currently a very expensive operation .. frees everything, resize file and remap.
      * Remapping involves rebuild of index.
      * @param required
      */
-    private void resizeStore(long required) {
+    public void resizeStore(long required, long maxgrowbytes) {
         if ( mappedFile == null )
             throw new RuntimeException("store is full. Required: "+required);
+        if ( required <= memory.length() )
+            return;
         mutationCount++;
         System.out.println("resizing underlying "+mappedFile+" to "+required+" numElem:"+numElem);
         long tim = System.currentTimeMillis();
@@ -286,7 +294,7 @@ private void resizeStore(long required) {
             File mf = new File(mappedFile);
             FileOutputStream f = new FileOutputStream(mf,true);
             long len = mf.length();
-            required = required + Math.min(required,MB*512);
+            required = required + Math.min(required,maxgrowbytes);
             byte[] toWrite = new byte[1000];
             long max = (required - len)/1000;
             for ( long i = 0; i < max+2; i++ ) {

diff --git a/src/main/java/org/nustaq/serialization/FSTClazzInfoRegistry.java b/src/main/java/org/nustaq/serialization/FSTClazzInfoRegistry.java
@@ -84,7 +84,14 @@ static void addAllReferencedClasses(Class cl, HashSet<String> names, HashSet<Str
             }
         }
 
-
+        Class[] classes = cl.getDeclaredClasses();
+        for (int i = 0; i < classes.length; i++) {
+            Class aClass = classes[i];
+            if ( ! aClass.isPrimitive() && ! aClass.isArray() ) {
+                names.add(aClass.getName());
+                addAllReferencedClasses(aClass, names,topLevelDone,filter);
+            }
+        }
 
         Class enclosingClass = cl.getEnclosingClass();
         if ( enclosingClass != null ) {

diff --git a/src/main/java/org/nustaq/serialization/FSTConfiguration.java b/src/main/java/org/nustaq/serialization/FSTConfiguration.java
@@ -158,7 +158,8 @@ public String getName() {
     public void setName(String name) {
         this.name = name;
     }
-/////////////////////////////////////
+
+    /////////////////////////////////////
     // cross platform stuff only
 
     int cpAttrIdCount = 0;
@@ -1124,6 +1125,16 @@ public Object asObject( byte b[] ) {
             return getObjectInput(b).readObject();
         } catch (Exception e) {
             System.out.println("unable to decode:" +new String(b,0,0,Math.min(b.length,100)) );
+            try {
+                String debug = new String(b, "UTF-8");
+            } catch (UnsupportedEncodingException e1) {
+                //
+            }
+            try {
+                getObjectInput(b).readObject();
+            } catch (Exception e1) {
+                // debug hook
+            }
             FSTUtil.<RuntimeException>rethrow(e);
         }
         return null;

diff --git a/src/main/java/org/nustaq/serialization/FSTEncoder.java b/src/main/java/org/nustaq/serialization/FSTEncoder.java
@@ -85,7 +85,8 @@ public interface FSTEncoder {
     // write a meta byte item. return true if encoder wrote full object (e.g. literal, primitive)
     boolean writeTag(byte tag, Object info, long somValue, Object toWrite, FSTObjectOutput oout) throws IOException;
 
-    void writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo);
+    // return true, if this already wrote everything
+    boolean writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo, Object value);
 
     void externalEnd(FSTClazzInfo clz); // demarkls the end of an externalizable or classes with serializer registered
 

diff --git a/src/main/java/org/nustaq/serialization/FSTObjectOutput.java b/src/main/java/org/nustaq/serialization/FSTObjectOutput.java
@@ -271,7 +271,6 @@ public void writeUTF(String s) throws IOException {
     /////////////////////////////////////////////////////
 
     public void writeObject(Object obj, Class... possibles) throws IOException {
-        curDepth++;
         if ( isCrossPlatform ) {
             writeObjectInternal(obj, null); // not supported cross platform
             return;
@@ -312,9 +311,6 @@ protected FSTClazzInfo.FSTFieldInfo getCachedFI( Class... possibles ) {
      * @throws IOException
      */
     public FSTClazzInfo writeObjectInternal(Object obj, FSTClazzInfo ci, Class... possibles) throws IOException {
-        if ( curDepth == 0 ) {
-            throw new RuntimeException("not intended to be called from external application. Use public writeObject instead");
-        }
         FSTClazzInfo.FSTFieldInfo info = getCachedFI(possibles);
         curDepth++;
         FSTClazzInfo fstClazzInfo = writeObjectWithContext(info, obj, ci);
@@ -600,7 +596,9 @@ protected void writeObjectFields(Object toWrite, FSTClazzInfo serializationInfo,
                     writeObjectFields(toWrite, serializationInfo, fieldInfo, i, subInfo.getVersion());
                     return;
                 }
-                getCodec().writeAttributeName(subInfo);
+                if ( getCodec().writeAttributeName(subInfo, toWrite) ) {
+                    continue;
+                }
                 if ( subInfo.isPrimitive() ) {
                     // speed safe
                     int integralType = subInfo.getIntegralType();

diff --git a/src/main/java/org/nustaq/serialization/FSTSerializerRegistry.java b/src/main/java/org/nustaq/serialization/FSTSerializerRegistry.java
@@ -28,6 +28,7 @@
  *
  */
 public class FSTSerializerRegistry {
+
     private FSTSerializerRegistryDelegate delegate;
 
     public static FSTObjectSerializer NULL = new NULLSerializer();

diff --git a/src/main/java/org/nustaq/serialization/coders/FSTBytezEncoder.java b/src/main/java/org/nustaq/serialization/coders/FSTBytezEncoder.java
@@ -179,8 +179,8 @@ public void writeFShort(short c) throws IOException {
         writePlainShort(c);
     }
 
-    public void writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo) {
-
+    public boolean writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo, Object value) {
+        return false;
     }
 
     public boolean writeTag(byte tag, Object info, long somValue, Object toWrite, FSTObjectOutput oout) throws IOException {

diff --git a/src/main/java/org/nustaq/serialization/coders/FSTJsonDecoder.java b/src/main/java/org/nustaq/serialization/coders/FSTJsonDecoder.java
@@ -7,6 +7,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
 import java.lang.reflect.Array;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -627,6 +628,13 @@ public void readObjectEnd() {
 
     @Override
     public Object coerceElement(Class arrType, Object value) {
+        if ( arrType == byte[].class && value instanceof String ) {
+            try {
+                return ((String) value).getBytes("UTF-8");
+            } catch (UnsupportedEncodingException e) {
+                e.printStackTrace();
+            }
+        }
         if ( value instanceof Number ) {
             Number n = (Number) value;
             if ( arrType == Byte.class ) {

diff --git a/src/main/java/org/nustaq/serialization/coders/FSTJsonEncoder.java b/src/main/java/org/nustaq/serialization/coders/FSTJsonEncoder.java
@@ -13,6 +13,8 @@
 import java.lang.reflect.Array;
 import java.util.Map;
 
+import static org.nustaq.serialization.FSTObjectOutput.STRING;
+
 /**
  * Created by ruedi on 20/05/15.
  *
@@ -386,7 +388,7 @@ protected String classToString(Class clz) {
     }
 
     @Override
-    public void writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo) {
+    public boolean writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo, Object outerObjectToWrite) {
         try {
             SerializedString bufferedName = (SerializedString) subInfo.getBufferedName();
             if ( bufferedName == null ) {
@@ -399,9 +401,22 @@ public void writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo) {
             else {
                 gen.writeFieldName(bufferedName);
             }
+            if ( subInfo.getField().isAnnotationPresent(JSONAsString.class) ) { // fixme: optimize
+                try {
+                    Object objectValue = subInfo.getObjectValue(outerObjectToWrite);
+                    if ( objectValue instanceof byte[] ) {
+                        String stringVal = new String((byte[]) objectValue, "UTF-8");
+                        writeStringUTF(stringVal);
+                        return true;
+                    }
+                } catch (IllegalAccessException e) {
+                    e.printStackTrace();
+                }
+            }
         } catch (IOException e) {
             FSTUtil.<RuntimeException>rethrow(e);
         }
+        return false;
     }
 
     @Override

diff --git a/src/main/java/org/nustaq/serialization/coders/FSTMinBinEncoder.java b/src/main/java/org/nustaq/serialization/coders/FSTMinBinEncoder.java
@@ -194,7 +194,7 @@ public void writeClass(FSTClazzInfo clInf) {
     }
 
     @Override
-    public void writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo) {
+    public boolean writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo, Object value) {
         byte[] bufferedName = (byte[]) subInfo.getBufferedName();
         if ( bufferedName != null ) {
             out.writeRaw(bufferedName,0,bufferedName.length);
@@ -209,6 +209,7 @@ public void writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo) {
             System.arraycopy(out.getBytez(),pos,bufferedName,0,len);
             subInfo.setBufferedName(bufferedName);
         }
+        return false;
     }
 
     @Override

diff --git a/src/main/java/org/nustaq/serialization/coders/FSTStreamEncoder.java b/src/main/java/org/nustaq/serialization/coders/FSTStreamEncoder.java
@@ -343,8 +343,8 @@ public void writeFShort(short c) throws IOException {
         }
     }
 
-    public void writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo) {
-
+    public boolean writeAttributeName(FSTClazzInfo.FSTFieldInfo subInfo, Object value) {
+        return false;
     }
 
     public boolean writeTag(byte tag, Object info, long somValue, Object toWrite, FSTObjectOutput oout) throws IOException {

diff --git a/src/main/java/org/nustaq/serialization/coders/JSONAsString.java b/src/main/java/org/nustaq/serialization/coders/JSONAsString.java
@@ -0,0 +1,19 @@
+package org.nustaq.serialization.coders;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Created by ruedi on 21.03.17.
+ *
+ * advice json serializer to convert annotated field to a string (e.g. byte or char arrays)
+ *
+ * !!!!!!!! supports byte[] only currently !!!!!!!!!!!!!!!
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.FIELD)
+
+public @interface JSONAsString {
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -28,6 +28,7 @@ @@
      *
      */
     public class FSTSerializerRegistry {
         private FSTSerializerRegistryDelegate delegate;
         public static FSTObjectSerializer NULL = new NULLSerializer();
@@ Expand Down @@