1 package org.djutils.data.xml;
2
3 import java.io.FileReader;
4 import java.io.FileWriter;
5 import java.io.IOException;
6 import java.io.Reader;
7 import java.io.Writer;
8 import java.util.ArrayList;
9 import java.util.List;
10 import java.util.function.Consumer;
11
12 import javax.xml.stream.XMLInputFactory;
13 import javax.xml.stream.XMLOutputFactory;
14 import javax.xml.stream.XMLStreamConstants;
15 import javax.xml.stream.XMLStreamException;
16 import javax.xml.stream.XMLStreamReader;
17 import javax.xml.stream.XMLStreamWriter;
18
19 import org.djutils.data.Column;
20 import org.djutils.data.ListTable;
21 import org.djutils.data.Row;
22 import org.djutils.data.Table;
23 import org.djutils.data.serialization.TextSerializationException;
24 import org.djutils.data.serialization.TextSerializer;
25 import org.djutils.exceptions.Throw;
26 import org.djutils.primitives.Primitive;
27
28 /**
29 * XmlData takes care of reading and writing of table data in XML format. The reader and writer use a streaming API to avoid
30 * excessive memory use. The class can be used, e.g., as follows:
31 *
32 * <pre>
33 * Table dataTable = new ListTable("data", "dataTable", columns);
34 * Writer writer = new FileWriter("c:/data/data.xml");
35 * XmlData.writeData(writer, dataTable);
36 * </pre>
37 *
38 * The XML document has the following structure:
39 *
40 * <pre>
41 * <xmldata>
42 * <table id="tableId" description="description" class="org.djutils.data.ListTable">
43 * <column nr="0" id="obsNr" description="observation nr" type="int"></column>
44 * <column nr="1" id="value" description="observation value" type="double"></column>
45 * <column nr="2" id="comment" description="comment" type="java.lang.String"></column>
46 * </table>
47 * <data>
48 * <record index="0">
49 * <value nr="0" content="2"></value>
50 * <value nr="1" content="18.6"></value>
51 * <value nr="2" content="normal"></value>
52 * </record>
53 * <record index="1">
54 * <value nr="0" content="4"></value>
55 * <value nr="1" content="36.18"></value>
56 * <value nr="2" content="normal"></value>
57 * </record>
58 * </data>
59 * </xmldata>
60 * </pre>
61 * <p>
62 * Copyright (c) 2020-2024 Delft University of Technology, PO Box 5, 2600 AA, Delft, the Netherlands. All rights reserved. <br>
63 * BSD-style license. See <a href="https://djutils.org/docs/current/djutils/licenses.html">DJUTILS License</a>.
64 * </p>
65 * @author <a href="https://github.com/averbraeck">Alexander Verbraeck</a>
66 * @author <a href="https://tudelft.nl/staff/p.knoppers-1">Peter Knoppers</a>
67 * @author <a href="https://dittlab.tudelft.nl">Wouter Schakel</a>
68 */
69 public final class XmlData
70 {
71 /**
72 * Utility class, no public constructor.
73 */
74 private XmlData()
75 {
76 // utility class
77 }
78
79 /**
80 * Write the data from the data table in XML format.
81 * @param writer Writer; the writer that writes the data, e.g. to a file
82 * @param dataTable Table; the data table to write
83 * @throws IOException on I/O error when writing the data
84 * @throws TextSerializationException on unknown data type for serialization
85 * @throws XMLStreamException on XML write error
86 */
87 public static void writeData(final Writer writer, final Table dataTable)
88 throws IOException, TextSerializationException, XMLStreamException
89 {
90 XMLStreamWriter xmlw = null;
91 try
92 {
93 XMLOutputFactory xmlOutputFactory = XMLOutputFactory.newInstance();
94 xmlw = xmlOutputFactory.createXMLStreamWriter(writer);
95
96 // XML header
97 xmlw.writeStartDocument();
98 xmlw.writeCharacters("\n");
99
100 // write the table metadata
101 xmlw.writeStartElement("xmldata");
102 xmlw.writeCharacters("\n");
103 xmlw.writeCharacters(" ");
104 xmlw.writeStartElement("table");
105 xmlw.writeAttribute("id", dataTable.getId());
106 xmlw.writeAttribute("description", dataTable.getDescription());
107 xmlw.writeAttribute("class", dataTable.getClass().getName());
108 xmlw.writeCharacters("\n");
109 int index = 0;
110 for (Column<?> column : dataTable.getColumns())
111 {
112 xmlw.writeCharacters(" ");
113 xmlw.writeEmptyElement("column");
114 xmlw.writeAttribute("nr", String.valueOf(index++));
115 xmlw.writeAttribute("id", column.getId());
116 xmlw.writeAttribute("description", column.getDescription());
117 xmlw.writeAttribute("type", column.getValueType().getName());
118 if (column.getUnit() != null)
119 {
120 xmlw.writeAttribute("unit", column.getUnit());
121 }
122 xmlw.writeCharacters("\n");
123 }
124 xmlw.writeCharacters(" ");
125 xmlw.writeEndElement(); // table
126 xmlw.writeCharacters("\n");
127
128 // initialize the serializers
129 TextSerializer<?>[] serializers = new TextSerializer[dataTable.getNumberOfColumns()];
130 for (int i = 0; i < dataTable.getNumberOfColumns(); i++)
131 {
132 Column<?> column = dataTable.getColumns().get(i);
133 serializers[i] = TextSerializer.resolve(column.getValueType());
134 }
135
136 // write the data
137 xmlw.writeCharacters(" ");
138 xmlw.writeStartElement("data");
139 xmlw.writeCharacters("\n");
140
141 // write the records
142 int recordNr = 0;
143 for (Row row : dataTable)
144 {
145 Object[] values = row.getValues();
146 xmlw.writeCharacters(" ");
147 xmlw.writeStartElement("row");
148 xmlw.writeAttribute("index", String.valueOf(recordNr++));
149 xmlw.writeCharacters("\n");
150 for (int i = 0; i < dataTable.getNumberOfColumns(); i++)
151 {
152 xmlw.writeCharacters(" ");
153 xmlw.writeEmptyElement("value");
154 xmlw.writeAttribute("nr", String.valueOf(i));
155 String content = TextSerializer.serialize(serializers[i], values[i], dataTable.getColumn(i).getUnit());
156 if (content != null)
157 {
158 xmlw.writeAttribute("content", content);
159 }
160 xmlw.writeCharacters("\n");
161 }
162 xmlw.writeCharacters(" ");
163 xmlw.writeEndElement(); // row
164 xmlw.writeCharacters("\n");
165 }
166
167 // end XML document
168 xmlw.writeCharacters(" ");
169 xmlw.writeEndElement(); // data
170 xmlw.writeCharacters("\n");
171 xmlw.writeEndElement(); // xmldata
172 xmlw.writeCharacters("\n");
173 xmlw.writeEndDocument();
174 }
175 finally
176 {
177 if (null != xmlw)
178 {
179 xmlw.close();
180 }
181 }
182 }
183
184 /**
185 * Write the data from the data table in XML format.
186 * @param filename String; the file name to write the data to
187 * @param dataTable Table; the data table to write
188 * @throws IOException on I/O error when writing the data
189 * @throws TextSerializationException on unknown data type for serialization
190 * @throws XMLStreamException on XML write error
191 */
192 public static void writeData(final String filename, final Table dataTable)
193 throws IOException, TextSerializationException, XMLStreamException
194 {
195 try (FileWriter fw = new FileWriter(filename))
196 {
197 writeData(fw, dataTable);
198 }
199 }
200
201 /**
202 * Read the data from the XML-file into the data table. Use the metadata to reconstruct the data table.
203 * @param reader Reader; the reader that can read the data, e.g. from a file
204 * @return dataTable the data table reconstructed from the meta data and filled with the data
205 * @throws IOException on I/O error when reading the data
206 * @throws TextSerializationException on unknown data type for serialization
207 * @throws XMLStreamException on XML read error
208 */
209 public static Table readData(final Reader reader) throws IOException, TextSerializationException, XMLStreamException
210 {
211 XMLStreamReader xmlr = null;
212 try
213 {
214 // read the metadata file and reconstruct the data table
215 XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance();
216 xmlr = xmlInputFactory.createXMLStreamReader(reader);
217
218 // wait for the xmldata tag
219 waitFor(xmlr, "xmldata");
220
221 // wait for the table tag
222 waitFor(xmlr, "table");
223 String[] tableProperties = getAttributes(xmlr, "id", "description", "class");
224
225 // column metadata
226 List<Column<?>> columns = new ArrayList<>();
227 int index = 0;
228 while (waitFor(xmlr, "column", "table"))
229 {
230 String[] columnProperties;
231 if (xmlr.getAttributeCount() == 5)
232 {
233 columnProperties = getAttributes(xmlr, "nr", "id", "description", "type", "unit");
234 }
235 else
236 {
237 // null unit
238 columnProperties = getAttributes(xmlr, "nr", "id", "description", "type");
239 }
240 if (Integer.valueOf(columnProperties[0]).intValue() != index)
241 {
242 throw new IOException("column nr not ok");
243 }
244 String type = columnProperties[3];
245 Class<?> valueClass = Primitive.forName(type);
246 if (valueClass == null)
247 {
248 try
249 {
250 valueClass = Class.forName(type);
251 }
252 catch (ClassNotFoundException exception)
253 {
254 throw new IOException("Could not find class " + type, exception);
255 }
256 }
257 Column<?> column = new Column<>(columnProperties[1], columnProperties[2], valueClass,
258 columnProperties.length >= 5 ? columnProperties[4] : null);
259 columns.add(column);
260 index++;
261 }
262
263 // create table
264 Table table;
265 Consumer<Object[]> unserializableTable;
266 if (tableProperties[2].equals(ListTable.class.getName()))
267 {
268 ListTable listTable = new ListTable(tableProperties[0], tableProperties[1], columns);
269 table = listTable;
270 unserializableTable = (
271 data
272 ) -> listTable.addRow(data);
273 }
274 else
275 {
276 // fallback
277 ListTable listTable = new ListTable(tableProperties[0], tableProperties[1], columns);
278 table = listTable;
279 unserializableTable = (
280 data
281 ) -> listTable.addRow(data);
282 }
283
284 // obtain the serializers
285 TextSerializer<?>[] serializers = new TextSerializer[table.getNumberOfColumns()];
286 for (int i = 0; i < table.getNumberOfColumns(); i++)
287 {
288 serializers[i] = TextSerializer.resolve(columns.get(i).getValueType());
289 }
290
291 // read the data file records
292 waitFor(xmlr, "data");
293 while (waitFor(xmlr, "row", "data"))
294 {
295 String[] data = new String[columns.size()];
296 while (waitFor(xmlr, "value", "row"))
297 {
298 if (xmlr.getAttributeCount() == 2)
299 {
300 String[] valueProperties = getAttributes(xmlr, "nr", "content");
301 data[Integer.valueOf(valueProperties[0]).intValue()] = valueProperties[1];
302 }
303 else
304 {
305 // null content
306 String[] valueProperties = getAttributes(xmlr, "nr");
307 data[Integer.valueOf(valueProperties[0]).intValue()] = null;
308 }
309 }
310 Object[] values = new Object[columns.size()];
311 for (int i = 0; i < values.length; i++)
312 {
313 values[i] = TextSerializer.deserialize(serializers[i], data[i], columns.get(i));
314 }
315 unserializableTable.accept(values); // addRow
316 }
317 return table;
318 }
319 finally
320 {
321 if (null != xmlr)
322 {
323 xmlr.close();
324 }
325 }
326 }
327
328 /**
329 * Read from the XML file until a START_ELEMENT with the id equal to the provided tag is encountered.
330 * @param xmlr XMLStreamReader; the XML stream reader
331 * @param tag String; the tag to retrieve
332 * @throws XMLStreamException on error reading from the XML stream
333 * @throws IOException when the stream ended without finding the tag
334 */
335 private static void waitFor(final XMLStreamReader xmlr, final String tag) throws XMLStreamException, IOException
336 {
337 while (xmlr.hasNext())
338 {
339 xmlr.next();
340 if (xmlr.getEventType() == XMLStreamConstants.START_ELEMENT)
341 {
342 if (xmlr.getLocalName().equals(tag))
343 {
344 return;
345 }
346 }
347 }
348 throw new IOException("Unexpected end of stream");
349 }
350
351 /**
352 * Read from the XML file until a START_ELEMENT with the id equal to the provided tag is encountered, or until the
353 * stopEndTag is reached. This can be used to get the starting tag in a repeat group. When the starting tag is found, the
354 * method returns true. When the end tag of the repeat group is found, false is returned.
355 * @param xmlr XMLStreamReader; the XML stream reader
356 * @param tag String; the tag to retrieve, usually a tag in a repeat group
357 * @param stopEndTag String; the tag to indicate the end of the repeat group
358 * @return true when the tag in the repeat group was found; false when the stop tag was found
359 * @throws XMLStreamException on error reading from the XML stream
360 * @throws IOException when the stream ended without finding the tag or the stop tag
361 */
362 private static boolean waitFor(final XMLStreamReader xmlr, final String tag, final String stopEndTag)
363 throws XMLStreamException, IOException
364 {
365 while (xmlr.hasNext())
366 {
367 xmlr.next();
368 if (xmlr.getEventType() == XMLStreamConstants.START_ELEMENT)
369 {
370 if (xmlr.getLocalName().equals(tag))
371 {
372 return true;
373 }
374 }
375 else if (xmlr.getEventType() == XMLStreamConstants.END_ELEMENT)
376 {
377 if (xmlr.getLocalName().equals(stopEndTag))
378 {
379 return false;
380 }
381 }
382 }
383 throw new IOException("Unexpected end of stream");
384 }
385
386 /**
387 * Read the attributes into an array and return the array. The position of each attribute is indicated by the vararg
388 * parameter 'attributes'.
389 * @param xmlr XMLStreamReader; the XML stream reader
390 * @param attributes String...; the attributes that are expected
391 * @return the array of atribute values, in the order of the vararg parameter 'attributes'
392 * @throws XMLStreamException on error reading from the XML stream
393 * @throws IOException when the current element does not contain the right (number of) attributes
394 */
395 private static String[] getAttributes(final XMLStreamReader xmlr, final String... attributes)
396 throws XMLStreamException, IOException
397 {
398 String[] result = new String[attributes.length];
399 int found = 0;
400 for (int i = 0; i < xmlr.getAttributeCount(); i++)
401 {
402 String localName = xmlr.getAttributeLocalName(i);
403 String value = xmlr.getAttributeValue(i);
404 for (int j = 0; j < attributes.length; j++)
405 {
406 if (localName.equals(attributes[j]))
407 {
408 result[j] = value;
409 found++;
410 }
411 }
412 }
413 Throw.when(found != attributes.length, IOException.class, "attribute data does not contain %d fields",
414 attributes.length);
415 return result;
416 }
417
418 /**
419 * Read the data from the XML-file into the data table. Use the metadata to reconstruct the data table.
420 * @param filename String; the file name to read the data from
421 * @return dataTable the data table reconstructed from the meta data and filled with the data
422 * @throws IOException on I/O error when reading the data
423 * @throws TextSerializationException on unknown data type for serialization
424 * @throws XMLStreamException on XML read error
425 */
426 public static Table readData(final String filename) throws IOException, TextSerializationException, XMLStreamException
427 {
428 try (FileReader fr = new FileReader(filename))
429 {
430 return readData(fr);
431 }
432 }
433
434 }