1 package org.djutils.data.json;
2
3 import java.io.FileReader;
4 import java.io.FileWriter;
5 import java.io.IOException;
6 import java.io.Reader;
7 import java.io.Writer;
8 import java.util.ArrayList;
9 import java.util.IllegalFormatException;
10 import java.util.List;
11 import java.util.function.Consumer;
12
13 import org.djutils.data.Column;
14 import org.djutils.data.ListTable;
15 import org.djutils.data.Row;
16 import org.djutils.data.Table;
17 import org.djutils.data.serialization.TextSerializationException;
18 import org.djutils.data.serialization.TextSerializer;
19 import org.djutils.exceptions.Throw;
20 import org.djutils.primitives.Primitive;
21
22 import com.google.gson.stream.JsonReader;
23 import com.google.gson.stream.JsonToken;
24 import com.google.gson.stream.JsonWriter;
25
26 /**
27 * JsonData takes care of reading and writing of table data in JSON format. The reader and writer use a streaming API to avoid
28 * excessive memory use. The class can be used, e.g., as follows:
29 *
30 * <pre>
31 * Table dataTable = new ListTable("data", "dataTable", columns);
32 * Writer writer = new FileWriter("c:/data/data.json");
33 * JsonData.writeData(writer, dataTable);
34 * </pre>
35 *
36 * The JSON document has the following structure:
37 *
38 * <pre>
39 * {
40 * "table": {
41 * "id": "tableId",
42 * "description": "table description",
43 * "class": "org.djutils.data.ListTable"",
44 * "columns": [
45 * {
46 * "nr": "0",
47 * "id": "time",
48 * "description": "time in [s]",
49 * "class": "org.djtils.vdouble.scalar.Time",
50 * },
51 * {
52 * "nr": "1",
53 * "id": "value",
54 * "description": "value [cm]",
55 * "class": "double",
56 * },
57 * {
58 * "nr": "2",
59 * "id": "comment",
60 * "description": "comment",
61 * "class": "java.lang.String",
62 * },
63 * ]
64 * },
65 * "data": [
66 * [ { "0" : "2" }, { "1": "14.6" }, { "2" : "normal" } ],
67 * [ { "0" : "4" }, { "1": "18.7" }, { "2" : "normal" } ],
68 * [ { "0" : "6" }, { "1": "21.3" }, { "2" : "abnormal" } ]
69 * ]
70 * }
71 * </pre>
72 * <p>
73 * Copyright (c) 2020-2024 Delft University of Technology, PO Box 5, 2600 AA, Delft, the Netherlands. All rights reserved. <br>
74 * BSD-style license. See <a href="https://djutils.org/docs/current/djutils/licenses.html">DJUTILS License</a>.
75 * </p>
76 * @author <a href="https://github.com/averbraeck">Alexander Verbraeck</a>
77 * @author <a href="https://tudelft.nl/staff/p.knoppers-1">Peter Knoppers</a>
78 * @author <a href="https://dittlab.tudelft.nl">Wouter Schakel</a>
79 */
80 public final class JsonData
81 {
82 /**
83 * Utility class, no public constructor.
84 */
85 private JsonData()
86 {
87 // utility class
88 }
89
90 /**
91 * Write the data from the data table in JSON format.
92 * @param writer Writer; the writer that writes the data, e.g. to a file
93 * @param dataTable Table; the data table to write
94 * @throws IOException on I/O error when writing the data
95 * @throws TextSerializationException on unknown data type for serialization
96 */
97 @SuppressWarnings("resource")
98 public static void writeData(final Writer writer, final Table dataTable) throws IOException, TextSerializationException
99 {
100 try (JsonWriter jw = new JsonWriter(writer);)
101 {
102
103 jw.setIndent(" ");
104
105 // write the table metadata
106 jw.beginObject();
107 jw.name("table").beginObject();
108 jw.name("id").value(dataTable.getId());
109 jw.name("description").value(dataTable.getDescription());
110 jw.name("class").value(dataTable.getClass().getName());
111 jw.name("columns").beginArray();
112 int index = 0;
113 for (Column<?> column : dataTable.getColumns())
114 {
115 jw.beginObject();
116 jw.name("nr").value(index++);
117 jw.name("id").value(column.getId());
118 jw.name("description").value(column.getDescription());
119 jw.name("type").value(column.getValueType().getName());
120 if (column.getUnit() != null)
121 {
122 jw.name("unit").value(column.getUnit());
123 }
124 jw.endObject();
125 }
126 jw.endArray(); // columns
127 jw.endObject(); // table
128
129 // initialize the serializers
130 TextSerializer<?>[] serializers = new TextSerializer[dataTable.getNumberOfColumns()];
131 for (int i = 0; i < dataTable.getNumberOfColumns(); i++)
132 {
133 Column<?> column = dataTable.getColumns().get(i);
134 serializers[i] = TextSerializer.resolve(column.getValueType());
135 }
136
137 // write the data
138 jw.name("data").beginArray();
139
140 // write the records
141 for (Row row : dataTable)
142 {
143 Object[] values = row.getValues();
144 jw.beginArray();
145 jw.setIndent("");
146 for (int i = 0; i < dataTable.getNumberOfColumns(); i++)
147 {
148 jw.beginObject().name(String.valueOf(i))
149 .value(TextSerializer.serialize(serializers[i], values[i], dataTable.getColumn(i).getUnit()))
150 .endObject();
151 }
152 jw.endArray(); // record
153 jw.setIndent(" ");
154 }
155
156 // end JSON document
157 jw.endArray(); // data array
158 jw.endObject(); // data
159 }
160 }
161
162 /**
163 * Write the data from the data table in JSON format.
164 * @param filename String; the file name to write the data to
165 * @param dataTable Table; the data table to write
166 * @throws IOException on I/O error when writing the data
167 * @throws TextSerializationException on unknown data type for serialization
168 */
169 public static void writeData(final String filename, final Table dataTable) throws IOException, TextSerializationException
170 {
171 FileWriter fw = null;
172 try
173 {
174 fw = new FileWriter(filename);
175 writeData(fw, dataTable);
176 }
177 finally
178 {
179 if (null != fw)
180 {
181 fw.close();
182 }
183 }
184 }
185
186 /**
187 * Read the data from the csv-file into the data table. Use the metadata to reconstruct the data table.
188 * @param reader Reader; the reader that can read the data, e.g. from a file
189 * @return dataTable the data table reconstructed from the meta data and filled with the data
190 * @throws IOException on I/O error when reading the data
191 * @throws TextSerializationException on unknown data type for serialization
192 */
193 public static Table readData(final Reader reader) throws IOException, TextSerializationException
194 {
195 try (JsonReader jr = new JsonReader(reader))
196 {
197 // read the metadata and reconstruct the data table
198 jr.beginObject();
199 readName(jr, "table");
200 jr.beginObject();
201 String[] tableProperties = new String[3];
202 tableProperties[0] = readValue(jr, "id");
203 tableProperties[1] = readValue(jr, "description");
204 tableProperties[2] = readValue(jr, "class");
205
206 // column metadata
207 List<Column<?>> columns = new ArrayList<>();
208 int index = 0;
209 readName(jr, "columns");
210 jr.beginArray();
211 while (jr.peek().equals(JsonToken.BEGIN_OBJECT))
212 {
213 String[] columnProperties = new String[5];
214 jr.beginObject();
215 columnProperties[0] = readValue(jr, "nr");
216 columnProperties[1] = readValue(jr, "id");
217 columnProperties[2] = readValue(jr, "description");
218 columnProperties[3] = readValue(jr, "type");
219 columnProperties[4] = jr.peek().equals(JsonToken.END_OBJECT) ? null : readValue(jr, "unit");
220 jr.endObject();
221
222 if (Integer.valueOf(columnProperties[0]).intValue() != index)
223 {
224 throw new IOException("column nr not ok");
225 }
226 String type = columnProperties[3];
227 Class<?> valueClass = Primitive.forName(type);
228 if (valueClass == null)
229 {
230 try
231 {
232 valueClass = Class.forName(type);
233 }
234 catch (ClassNotFoundException exception)
235 {
236 throw new IOException("Could not find class " + type, exception);
237 }
238 }
239 Column<?> column = new Column<>(columnProperties[1], columnProperties[2], valueClass, columnProperties[4]);
240 columns.add(column);
241 index++;
242 }
243 jr.endArray(); // columns
244 jr.endObject(); // table
245
246 // create table
247 Table table;
248 Consumer<Object[]> unserializableTable;
249 if (tableProperties[2].equals(ListTable.class.getName()))
250 {
251 ListTable listTable = new ListTable(tableProperties[0], tableProperties[1], columns);
252 table = listTable;
253 unserializableTable = (
254 data
255 ) -> listTable.addRow(data);
256 }
257 else
258 {
259 // fallback
260 ListTable listTable = new ListTable(tableProperties[0], tableProperties[1], columns);
261 table = listTable;
262 unserializableTable = (
263 data
264 ) -> listTable.addRow(data);
265 }
266
267 // obtain the serializers
268 TextSerializer<?>[] serializers = new TextSerializer[table.getNumberOfColumns()];
269 for (int i = 0; i < table.getNumberOfColumns(); i++)
270 {
271 serializers[i] = TextSerializer.resolve(columns.get(i).getValueType());
272 }
273
274 // read the data file records
275 readName(jr, "data");
276 jr.beginArray();
277 while (jr.peek().equals(JsonToken.BEGIN_ARRAY))
278 {
279 Object[] values = new Object[columns.size()];
280 jr.beginArray();
281 for (int i = 0; i < table.getNumberOfColumns(); i++)
282 {
283 jr.beginObject();
284 values[i] = TextSerializer.deserialize(serializers[i], readValue(jr, "" + i), columns.get(i));
285 jr.endObject();
286 }
287 jr.endArray(); // row
288 unserializableTable.accept(values); // addRow
289 }
290
291 // end JSON document
292 jr.endArray(); // data array
293 jr.endObject(); // data
294 return table;
295 }
296 }
297
298 /**
299 * Read a name - value pair from the JSON file where name has to match the given tag name.
300 * @param jr JsonReader; the JSON stream reader
301 * @param tag String; the tag to retrieve
302 * @return the value belonging to the tag
303 * @throws IllegalFormatException when the next element in the file did not contain the right tag
304 * @throws IOException when reading from the stream raises an exception
305 */
306 private static String readValue(final JsonReader jr, final String tag) throws IllegalFormatException, IOException
307 {
308 Throw.when(!jr.nextName().equals(tag), IllegalFormatException.class, "readValue: no %s object", tag);
309 if (jr.peek().equals(JsonToken.NULL))
310 {
311 jr.nextNull();
312 return null;
313 }
314 return jr.nextString();
315 }
316
317 /**
318 * Read a name -from the JSON file where name has to match the given tag name.
319 * @param jr JsonReader; the JSON stream reader
320 * @param tag String; the tag to retrieve
321 * @throws IllegalFormatException when the next element in the file did not contain the right tag
322 * @throws IOException when reading from the stream raises an exception
323 */
324 private static void readName(final JsonReader jr, final String tag) throws IllegalFormatException, IOException
325 {
326 Throw.when(!jr.nextName().equals(tag), IllegalFormatException.class, "readName: no %s object", tag);
327 }
328
329 /**
330 * Read the data from the csv-file into the data table. Use the metadata to reconstruct the data table.
331 * @param filename String; the file name to read the data from
332 * @return dataTable the data table reconstructed from the meta data and filled with the data
333 * @throws IOException on I/O error when reading the data
334 * @throws TextSerializationException on unknown data type for serialization
335 */
336 public static Table readData(final String filename) throws IOException, TextSerializationException
337 {
338 try (FileReader fr = new FileReader(filename);)
339 {
340 return readData(fr);
341 }
342 }
343
344 }