json serialization in progress

71219463 · Denis Arrivault · b9ace487 · 71219463 · 71219463 · 71219463
Commit 71219463 authored 7 years ago by Denis Arrivault
--- a/copyrightstamp.txt
+++ b/copyrightstamp.txt
@@ -10,7 +10,7 @@ Contributors:
 ------------
 * François Denis <francois.denis_AT_lif.univ-mrs.fr>
-* Rémy Eyraud <remy.eyraud_AT_lif.univ-mrs.fr>
+* Rémi Eyraud <remy.eyraud_AT_lif.univ-mrs.fr>
 * Denis Arrivault <contact.dev_AT_lif.univ-mrs.fr>
 * Dominique Benielli <dominique.benielli_AT_univ-amu.fr>

--- a/examples/json_save.ipynb
+++ b/examples/json_save.ipynb
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Start Hankel matrix computation\n",
+      "End of Hankel matrix computation\n",
+      "Start Building Automaton from Hankel matrix\n",
+      "End of Automaton computation\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Spectral(lcolumns=7, lrows=7, mode_quiet=False, partial=True, rank=5,\n",
+       "     smooth_method='none', sparse=True, version='classic')"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from splearn.datasets.base import load_data_sample\n",
+    "from splearn.tests.datasets.get_dataset_path import get_dataset_path\n",
+    "from splearn import Spectral\n",
+    "train_file = '3.pautomac_light.train'\n",
+    "data = load_data_sample(adr=get_dataset_path(train_file))\n",
+    "sp = Spectral()\n",
+    "sp.fit(X=data.data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "AttributeError",
+     "evalue": "'dict' object has no attribute 'iteritems'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-2-2625a21006a9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0msp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautomaton\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite_json\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"test.json\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;32m~/Codes/scikit-splearn/splearn/automaton.py\u001b[0m in \u001b[0;36mwrite_json\u001b[0;34m(self, jsonFileName)\u001b[0m\n\u001b[1;32m    695\u001b[0m                 \"transitions\":self.transitions, \"type\":self.type}\n\u001b[1;32m    696\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjsonFileName\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'w'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0moutfile\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 697\u001b[0;31m             \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSerializer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata_to_json\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutfile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    698\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    699\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mread_json\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjsonFileName\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Codes/scikit-splearn/splearn/serializer.py\u001b[0m in \u001b[0;36mdata_to_json\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m     45\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mdata_to_json\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 47\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdumps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__serialize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     48\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     49\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mjson_to_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Codes/scikit-splearn/splearn/serializer.py\u001b[0m in \u001b[0;36m__serialize\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__serialize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miteritems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m\"py/dict\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__serialize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__serialize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miteritems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;31mAttributeError\u001b[0m: 'dict' object has no attribute 'iteritems'"
+     ]
+    }
+   ],
+   "source": [
+    "sp.automaton.write_json(\"test.json\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
+%% Cell type:code id: tags:
+``` python
+from splearn.datasets.base import load_data_sample
+from splearn.tests.datasets.get_dataset_path import get_dataset_path
+from splearn import Spectral
+train_file = '3.pautomac_light.train'
+data = load_data_sample(adr=get_dataset_path(train_file))
+sp = Spectral()
+sp.fit(X=data.data)
+```
+%% Output
+    Start Hankel matrix computation
+    End of Hankel matrix computation
+    Start Building Automaton from Hankel matrix
+    End of Automaton computation
+    Spectral(lcolumns=7, lrows=7, mode_quiet=False, partial=True, rank=5,
+         smooth_method='none', sparse=True, version='classic')
+%% Cell type:code id: tags:
+``` python
+sp.automaton.write_json("test.json")
+```
+%% Output
+    ---------------------------------------------------------------------------
+    AttributeError                            Traceback (most recent call last)
+    <ipython-input-2-2625a21006a9> in <module>()
+    ----> 1 sp.automaton.write_json("test.json")
+    ~/Codes/scikit-splearn/splearn/automaton.py in write_json(self, jsonFileName)
+        695                 "transitions":self.transitions, "type":self.type}
+        696         with open(jsonFileName, 'w') as outfile:
+    --> 697             json.dump(Serializer().data_to_json(data), outfile)
+        698
+        699     def read_json(self, jsonFileName):
+    ~/Codes/scikit-splearn/splearn/serializer.py in data_to_json(self, data)
+         45
+         46     def data_to_json(self, data):
+    ---> 47         return json.dumps(self.__serialize(data))
+         48
+         49     def json_to_data(self, s):
+    ~/Codes/scikit-splearn/splearn/serializer.py in __serialize(self, data)
+         20         if isinstance(data, dict):
+         21             if all(isinstance(k, str) for k in data):
+    ---> 22                 return {k: self.__serialize(v) for k, v in data.iteritems()}
+         23             return {"py/dict": [[self.__serialize(k), self.__serialize(v)] for k, v in data.iteritems()]}
+         24         if isinstance(data, tuple):
+    AttributeError: 'dict' object has no attribute 'iteritems'
+%% Cell type:code id: tags:
+``` python
+```
--- a/examples/json_save.py
+++ b/examples/json_save.py
+from splearn.datasets.base import load_data_sample
+from splearn.tests.datasets.get_dataset_path import get_dataset_path
+from splearn import Spectral
+from splearn.automaton import Automaton
+train_file = '3.pautomac_light.train'
+data = load_data_sample(adr=get_dataset_path(train_file))
+sp = Spectral()
+sp.fit(X=data.data)
+sp.automaton.write_json("test.json")
+A = Automaton(1,1,[]).read_json("test.json")
+print(A)
--- a/examples/test.json
+++ b/examples/test.json
+{"nbL": 4, "nbS": 5, "initial": {"py/numpy.ndarray": {"values": [-0.0004934419970497512, 0.0030634697107912346, -0.044073932015580415, -0.1077770261654714, -0.0866391379316952], "dtype": "float64"}}, "final": {"py/numpy.ndarray": {"values": [0.07757136847945045, -0.024220294003132026, -0.4468125366321221, 0.627732084089759, -0.554674433356224], "dtype": "float64"}}, "transitions": [{"py/numpy.ndarray": {"values": [[0.04512120959511772, -0.24038969827844062, 0.34944999592135334, -0.2811680730534579, -0.21402523377497645], [0.0692580056243761, -0.30062293462829204, 0.20641375368520157, -0.14960814319756124, -0.5580573163749153], [0.02980115192176571, -0.13866480809160409, 0.18362212572805459, -0.20969545230657607, -0.14481622025561292], [0.005699344003198349, -0.023385825120201414, -0.06600665373981851, 0.10749935271466007, -0.15103654604159977], [-0.02008655193147911, 0.09026347555230492, -0.005525585655539262, -0.031355317090308935, 0.2432902242047721]], "dtype": "float64"}}, {"py/numpy.ndarray": {"values": [[0.0774477207917058, 0.09007073705762021, -0.3047220063293013, 0.2767624549859105, 0.20289396030628148], [-0.09902980483670844, -0.08061846818727973, 0.25853170692250554, -0.12086330214608881, -0.11085207725068251], [-0.061710792028537534, -0.06244151779954751, 0.12007654564862075, 0.0025063746277943564, -0.1567967473145572], [-0.002736973749965403, -0.009005721984277787, -0.00046003295909181354, -0.008550426472005344, -0.053754646789681754], [0.030987327588710728, 0.03972680066723246, -0.04997113350910248, 0.0035769411874962344, 0.1418257620585633]], "dtype": "float64"}}, {"py/numpy.ndarray": {"values": [[-0.06791915236220235, -0.11357937659088102, 0.37955392604054394, -0.21784979894046635, -0.22977695089938127], [0.11596642335411328, 0.14914956804629287, -0.13357508376686902, -0.008916063072034974, 0.3484153673774836], [0.011730817547426673, 0.019273800531955612, 0.0414265834586712, -0.035346588560982, 0.02316491010895583], [0.007328911075541707, 0.005536509132796312, -0.022456082950666856, 0.03611543477693187, -0.038514339001406585], [-0.010589894686551544, -0.010626616553723532, -0.000543105645661794, -0.025567476700160314, 0.04984888818929034]], "dtype": "float64"}}, {"py/numpy.ndarray": {"values": [[0.07276211427780357, -0.0157195576855797, 0.07428592814590385, -0.10369861539249735, 0.024753473688328077], [-0.05607105449779142, -0.08896207276035666, 0.27638225397521243, -0.2371125582838589, 0.07372294122306285], [-0.007391294007753122, -0.048741797963875705, -0.6291239733858526, 0.46816276521577677, 0.09251699239093385], [-0.007110224931878467, -0.05623317735898056, -0.36606658567620365, -0.013297798115225407, 0.6491033177492604], [0.002335515008556511, -0.021561151264484414, 0.09096243479437888, -0.38438823493062646, 0.6616477207948602]], "dtype": "float64"}}], "type": "classic"}
\ No newline at end of file
--- a/splearn/automaton.py
+++ b/splearn/automaton.py
@@ -3,6 +3,7 @@
 """
 import numpy as np
+from splearn.serializer import Serializer
 class Automaton(object):
    """ Define an automaton with parameters
@@ -687,3 +688,25 @@ class Automaton(object):
                        out += label + "\"]\n"
        out += "}\n"
        return out
+    def write_json(self, jsonFileName):
+        data = {"nbL":self.nbL, "nbS":self.nbS, "initial":self.initial, "final":self.final, 
+                "transitions":self.transitions, "type":self.type}
+        data_str = Serializer().data_to_json(data)
+        with open(jsonFileName, 'w') as outfile:
+            outfile.write(data_str)
+    def read_json(self, jsonFileName):
+        with open(jsonFileName) as infile:
+            datastr = infile.read()
+        data = Serializer().json_to_data(datastr)
+        print(data)
+        keys = {"nbL", "nbS", "initial", "final", "transitions", "type"}
+        if not keys.issubset(set(data.keys())):
+            raise ValueError("The input json file should contain the following keys : \"" + '\", \"'.join(keys) + "\"")
+        self = Automaton(nbL=data["nbL"], nbS=data["nbS"], initial=data["initial"], final=data["final"],
+                         transitions=data["transitions"], type=data["type"])
--- a/splearn/serializer.py
+++ b/splearn/serializer.py
+'''
+Created on 6 févr. 2018
+@author: arrivault
+'''
+import numpy as np
+import json
+class Serializer(object):
+    '''
+    classdocs
+    '''
+    def __serialize(self, data):
+        if data is None or isinstance(data, (bool, int, float, str)):
+            return data
+        if isinstance(data, list):
+            return [self.__serialize(val) for val in data]
+        if isinstance(data, dict):
+            if all(isinstance(k, str) for k in data):
+                return {k: self.__serialize(v) for k, v in data.items()}
+            return {"py/dict": [[self.__serialize(k), self.__serialize(v)] for k, v in data.items()]}
+        if isinstance(data, tuple):
+            return {"py/tuple": [self.__serialize(val) for val in data]}
+        if isinstance(data, set):
+            return {"py/set": [self.__serialize(val) for val in data]}
+        if isinstance(data, np.ndarray):
+            return {"py/numpy.ndarray": {
+                "values": data.tolist(),
+                "dtype":  str(data.dtype)}}
+        raise TypeError("Type %s not data-serializable" % type(data))
+    def __restore(self, dct):
+        if "py/dict" in dct:
+            return dict(dct["py/dict"])
+        if "py/tuple" in dct:
+            return tuple(dct["py/tuple"])
+        if "py/set" in dct:
+            return set(dct["py/set"])
+        if "py/numpy.ndarray" in dct:
+            data = dct["py/numpy.ndarray"]
+            return np.array(data["values"], dtype=data["dtype"])
+        return dct
+    def data_to_json(self, data):
+        return json.dumps(self.__serialize(data))
+    def json_to_data(self, s):
+        return json.loads(s, object_hook=self.__restore)
--- a/test.json
+++ b/test.json
+{"nbL": 4, "nbS": 5, "initial": {"py/numpy.ndarray": {"values": [-0.0004934419970497512, 0.0030634697107912346, -0.044073932015580415, -0.1077770261654714, -0.0866391379316952], "dtype": "float64"}}, "final": {"py/numpy.ndarray": {"values": [0.07757136847945045, -0.024220294003132026, -0.4468125366321221, 0.627732084089759, -0.554674433356224], "dtype": "float64"}}, "transitions": [{"py/numpy.ndarray": {"values": [[0.04512120959511772, -0.24038969827844062, 0.34944999592135334, -0.2811680730534579, -0.21402523377497645], [0.0692580056243761, -0.30062293462829204, 0.20641375368520157, -0.14960814319756124, -0.5580573163749153], [0.02980115192176571, -0.13866480809160409, 0.18362212572805459, -0.20969545230657607, -0.14481622025561292], [0.005699344003198349, -0.023385825120201414, -0.06600665373981851, 0.10749935271466007, -0.15103654604159977], [-0.02008655193147911, 0.09026347555230492, -0.005525585655539262, -0.031355317090308935, 0.2432902242047721]], "dtype": "float64"}}, {"py/numpy.ndarray": {"values": [[0.0774477207917058, 0.09007073705762021, -0.3047220063293013, 0.2767624549859105, 0.20289396030628148], [-0.09902980483670844, -0.08061846818727973, 0.25853170692250554, -0.12086330214608881, -0.11085207725068251], [-0.061710792028537534, -0.06244151779954751, 0.12007654564862075, 0.0025063746277943564, -0.1567967473145572], [-0.002736973749965403, -0.009005721984277787, -0.00046003295909181354, -0.008550426472005344, -0.053754646789681754], [0.030987327588710728, 0.03972680066723246, -0.04997113350910248, 0.0035769411874962344, 0.1418257620585633]], "dtype": "float64"}}, {"py/numpy.ndarray": {"values": [[-0.06791915236220235, -0.11357937659088102, 0.37955392604054394, -0.21784979894046635, -0.22977695089938127], [0.11596642335411328, 0.14914956804629287, -0.13357508376686902, -0.008916063072034974, 0.3484153673774836], [0.011730817547426673, 0.019273800531955612, 0.0414265834586712, -0.035346588560982, 0.02316491010895583], [0.007328911075541707, 0.005536509132796312, -0.022456082950666856, 0.03611543477693187, -0.038514339001406585], [-0.010589894686551544, -0.010626616553723532, -0.000543105645661794, -0.025567476700160314, 0.04984888818929034]], "dtype": "float64"}}, {"py/numpy.ndarray": {"values": [[0.07276211427780357, -0.0157195576855797, 0.07428592814590385, -0.10369861539249735, 0.024753473688328077], [-0.05607105449779142, -0.08896207276035666, 0.27638225397521243, -0.2371125582838589, 0.07372294122306285], [-0.007391294007753122, -0.048741797963875705, -0.6291239733858526, 0.46816276521577677, 0.09251699239093385], [-0.007110224931878467, -0.05623317735898056, -0.36606658567620365, -0.013297798115225407, 0.6491033177492604], [0.002335515008556511, -0.021561151264484414, 0.09096243479437888, -0.38438823493062646, 0.6616477207948602]], "dtype": "float64"}}], "type": "classic"}
\ No newline at end of file