1 package org.djutils.data.json;
2
3 import java.io.FileReader;
4 import java.io.FileWriter;
5 import java.io.IOException;
6 import java.io.Reader;
7 import java.io.Writer;
8 import java.util.ArrayList;
9 import java.util.IllegalFormatException;
10 import java.util.List;
11
12 import org.djutils.data.DataColumn;
13 import org.djutils.data.DataRecord;
14 import org.djutils.data.DataTable;
15 import org.djutils.data.ListDataTable;
16 import org.djutils.data.SimpleDataColumn;
17 import org.djutils.data.serialization.TextSerializationException;
18 import org.djutils.data.serialization.TextSerializer;
19 import org.djutils.exceptions.Throw;
20 import org.djutils.primitives.Primitive;
21
22 import com.google.gson.stream.JsonReader;
23 import com.google.gson.stream.JsonToken;
24 import com.google.gson.stream.JsonWriter;
25
26 /**
27 * JSONData takes care of reading and writing of table data in JSON format. The reader and writer use a streaming API to avoid
28 * excessive memory use. The class can be used, e.g., as follows:
29 *
30 * <pre>
31 * DataTable dataTable = new ListDataTable("data", "dataTable", columns);
32 * Writer writer = new FileWriter("c:/data/data.json");
33 * JSONData.writeData(writer, dataTable);
34 * </pre>
35 *
36 * The JSON document has the following structure:
37 *
38 * <pre>
39 * {
40 * "table": {
41 * "id": "tableId",
42 * "description": "table description",
43 * "class": "org.djutils.data.ListDataTable"",
44 * "columns": [
45 * {
46 * "nr": "0",
47 * "id": "time",
48 * "description": "time in [s]",
49 * "class": "org.djtils.vdouble.scalar.Time",
50 * },
51 * {
52 * "nr": "1",
53 * "id": "value",
54 * "description": "value [cm]",
55 * "class": "double",
56 * },
57 * {
58 * "nr": "2",
59 * "id": "comment",
60 * "description": "comment",
61 * "class": "java.lang.String",
62 * },
63 * ]
64 * },
65 * "data": [
66 * [ { "0" : "2" }, { "1": "14.6" }, { "2" : "normal" } ],
67 * [ { "0" : "4" }, { "1": "18.7" }, { "2" : "normal" } ],
68 * [ { "0" : "6" }, { "1": "21.3" }, { "2" : "abnormal" } ]
69 * ]
70 * }
71 * </pre>
72 *
73 * Copyright (c) 2020-2021 Delft University of Technology, Jaffalaan 5, 2628 BX Delft, the Netherlands. All rights reserved. See
74 * for project information <a href="https://djutils.org" target="_blank"> https://djutils.org</a>. The DJUTILS project is
75 * distributed under a three-clause BSD-style license, which can be found at
76 * <a href="https://djutils.org/docs/license.html" target="_blank"> https://djutils.org/docs/license.html</a>. <br>
77 * @author <a href="https://www.tudelft.nl/averbraeck">Alexander Verbraeck</a>
78 * @author <a href="https://www.tudelft.nl/pknoppers">Peter Knoppers</a>
79 * @author <a href="http://www.transport.citg.tudelft.nl">Wouter Schakel</a>
80 */
81 public final class JSONData
82 {
83 /**
84 * Utility class, no public constructor.
85 */
86 private JSONData()
87 {
88 // utility class
89 }
90
91 /**
92 * Write the data from the data table in JSON format.
93 * @param writer Writer; the writer that writes the data, e.g. to a file
94 * @param dataTable DataTable; the data table to write
95 * @throws IOException on I/O error when writing the data
96 * @throws TextSerializationException on unknown data type for serialization
97 */
98 @SuppressWarnings("resource")
99 public static void writeData(final Writer writer, final DataTable dataTable) throws IOException, TextSerializationException
100 {
101 JsonWriter jw = null;
102 try
103 {
104 jw = new JsonWriter(writer);
105 jw.setIndent(" ");
106
107 // write the table metadata
108 jw.beginObject();
109 jw.name("table").beginObject();
110 jw.name("id").value(dataTable.getId());
111 jw.name("description").value(dataTable.getDescription());
112 jw.name("class").value(dataTable.getClass().getName());
113 jw.name("columns").beginArray();
114 int index = 0;
115 for (DataColumn<?> column : dataTable.getColumns())
116 {
117 jw.beginObject();
118 jw.name("nr").value(index++);
119 jw.name("id").value(column.getId());
120 jw.name("description").value(column.getDescription());
121 jw.name("type").value(column.getValueType().getName());
122 jw.endObject();
123 }
124 jw.endArray(); // columns
125 jw.endObject(); // table
126
127 // initialize the serializers
128 TextSerializer<?>[] serializers = new TextSerializer[dataTable.getNumberOfColumns()];
129 for (int i = 0; i < dataTable.getNumberOfColumns(); i++)
130 {
131 DataColumn<?> column = dataTable.getColumns().get(i);
132 serializers[i] = TextSerializer.resolve(column.getValueType());
133 }
134
135 // write the data
136 jw.name("data").beginArray();
137
138 // write the records
139 for (DataRecord record : dataTable)
140 {
141 Object[] values = record.getValues();
142 jw.beginArray();
143 jw.setIndent("");
144 for (int i = 0; i < dataTable.getNumberOfColumns(); i++)
145 {
146 jw.beginObject().name(String.valueOf(i)).value(serializers[i].serialize(values[i])).endObject();
147 }
148 jw.endArray(); // record
149 jw.setIndent(" ");
150 }
151
152 // end JSON document
153 jw.endArray(); // data array
154 jw.endObject(); // data
155 }
156 finally
157 {
158 if (null != jw)
159 {
160 jw.close();
161 }
162 }
163 }
164
165 /**
166 * Write the data from the data table in JSON format.
167 * @param filename String; the file name to write the data to
168 * @param dataTable DataTable; the data table to write
169 * @throws IOException on I/O error when writing the data
170 * @throws TextSerializationException on unknown data type for serialization
171 */
172 public static void writeData(final String filename, final DataTable dataTable)
173 throws IOException, TextSerializationException
174 {
175 FileWriter fw = null;
176 try
177 {
178 fw = new FileWriter(filename);
179 writeData(fw, dataTable);
180 }
181 finally
182 {
183 if (null != fw)
184 {
185 fw.close();
186 }
187 }
188 }
189
190 /**
191 * Read the data from the csv-file into the data table. Use the metadata to reconstruct the data table.
192 * @param reader Reader; the reader that can read the data, e.g. from a file
193 * @return dataTable the data table reconstructed from the meta data and filled with the data
194 * @throws IOException on I/O error when reading the data
195 * @throws TextSerializationException on unknown data type for serialization
196 */
197 public static DataTable readData(final Reader reader) throws IOException, TextSerializationException
198 {
199 JsonReader jr = null;
200 try
201 {
202 // read the metadata and reconstruct the data table
203 jr = new JsonReader(reader);
204 jr.beginObject();
205 readName(jr, "table");
206 jr.beginObject();
207 String[] tableProperties = new String[3];
208 tableProperties[0] = readValue(jr, "id");
209 tableProperties[1] = readValue(jr, "description");
210 tableProperties[2] = readValue(jr, "class");
211 Throw.when(!tableProperties[2].endsWith("ListDataTable"), IOException.class,
212 "Currently, this method can only recreate a ListDataTable");
213
214 // column metadata
215 List<DataColumn<?>> columns = new ArrayList<>();
216 int index = 0;
217 readName(jr, "columns");
218 jr.beginArray();
219 while (jr.peek().equals(JsonToken.BEGIN_OBJECT))
220 {
221 String[] columnProperties = new String[4];
222 jr.beginObject();
223 columnProperties[0] = readValue(jr, "nr");
224 columnProperties[1] = readValue(jr, "id");
225 columnProperties[2] = readValue(jr, "description");
226 columnProperties[3] = readValue(jr, "type");
227 jr.endObject();
228
229 if (Integer.valueOf(columnProperties[0]).intValue() != index)
230 {
231 throw new IOException("column nr not ok");
232 }
233 String type = columnProperties[3];
234 Class<?> valueClass = Primitive.forName(type);
235 if (valueClass == null)
236 {
237 try
238 {
239 valueClass = Class.forName(type);
240 }
241 catch (ClassNotFoundException exception)
242 {
243 throw new IOException("Could not find class " + type, exception);
244 }
245 }
246 @SuppressWarnings({"rawtypes", "unchecked"})
247 DataColumn<?> column = new SimpleDataColumn(columnProperties[1], columnProperties[2], valueClass);
248 columns.add(column);
249 index++;
250 }
251 jr.endArray(); // columns
252 jr.endObject(); // table
253
254 ListDataTablestDataTable">ListDataTable dataTable = new ListDataTable(tableProperties[0], tableProperties[1], columns);
255
256 // obtain the serializers
257 TextSerializer<?>[] serializers = new TextSerializer[dataTable.getNumberOfColumns()];
258 for (int i = 0; i < dataTable.getNumberOfColumns(); i++)
259 {
260 DataColumn<?> column = dataTable.getColumns().get(i);
261 serializers[i] = TextSerializer.resolve(column.getValueType());
262 }
263
264 // read the data file records
265 readName(jr, "data");
266 jr.beginArray();
267 while (jr.peek().equals(JsonToken.BEGIN_ARRAY))
268 {
269 Object[] values = new Object[columns.size()];
270 jr.beginArray();
271 for (int i = 0; i < dataTable.getNumberOfColumns(); i++)
272 {
273 jr.beginObject();
274 values[i] = serializers[i].deserialize(readValue(jr, "" + i));
275 jr.endObject();
276 }
277 jr.endArray(); // record
278 dataTable.addRecord(values);
279 }
280
281 // end JSON document
282 jr.endArray(); // data array
283 jr.endObject(); // data
284 return dataTable;
285 }
286 finally
287 {
288 if (null != jr)
289 {
290 jr.close();
291 }
292 }
293 }
294
295 /**
296 * Read a name - value pair from the JSON file where name has to match the given tag name.
297 * @param jr JsonReader; the JSON stream reader
298 * @param tag String; the tag to retrieve
299 * @return the value belonging to the tag
300 * @throws IllegalFormatException when the next element in the file did not contain the right tag
301 * @throws IOException when reading from the stream raises an exception
302 */
303 private static String readValue(final JsonReader jr, final String tag) throws IllegalFormatException, IOException
304 {
305 Throw.when(!jr.nextName().equals(tag), IllegalFormatException.class, "readValue: no %s object", tag);
306 return jr.nextString();
307 }
308
309 /**
310 * Read a name -from the JSON file where name has to match the given tag name.
311 * @param jr JsonReader; the JSON stream reader
312 * @param tag String; the tag to retrieve
313 * @throws IllegalFormatException when the next element in the file did not contain the right tag
314 * @throws IOException when reading from the stream raises an exception
315 */
316 private static void readName(final JsonReader jr, final String tag) throws IllegalFormatException, IOException
317 {
318 Throw.when(!jr.nextName().equals(tag), IllegalFormatException.class, "readName: no %s object", tag);
319 }
320
321 /**
322 * Read the data from the csv-file into the data table. Use the metadata to reconstruct the data table.
323 * @param filename String; the file name to read the data from
324 * @return dataTable the data table reconstructed from the meta data and filled with the data
325 * @throws IOException on I/O error when reading the data
326 * @throws TextSerializationException on unknown data type for serialization
327 */
328 public static DataTable readData(final String filename) throws IOException, TextSerializationException
329 {
330 FileReader fr = null;
331 try
332 {
333 fr = new FileReader(filename);
334 return readData(fr);
335 }
336 finally
337 {
338 if (null != fr)
339 {
340 fr.close();
341 }
342 }
343 }
344
345 }