1 package org.djutils.data.xml;
2
3 import java.io.FileReader;
4 import java.io.FileWriter;
5 import java.io.IOException;
6 import java.io.Reader;
7 import java.io.Writer;
8 import java.util.ArrayList;
9 import java.util.List;
10
11 import javax.xml.stream.XMLInputFactory;
12 import javax.xml.stream.XMLOutputFactory;
13 import javax.xml.stream.XMLStreamConstants;
14 import javax.xml.stream.XMLStreamException;
15 import javax.xml.stream.XMLStreamReader;
16 import javax.xml.stream.XMLStreamWriter;
17
18 import org.djutils.data.DataColumn;
19 import org.djutils.data.DataRecord;
20 import org.djutils.data.DataTable;
21 import org.djutils.data.ListDataTable;
22 import org.djutils.data.SimpleDataColumn;
23 import org.djutils.data.serialization.TextSerializationException;
24 import org.djutils.data.serialization.TextSerializer;
25 import org.djutils.exceptions.Throw;
26 import org.djutils.primitives.Primitive;
27
28 /**
29 * XMLData takes care of reading and writing of table data in XML format. The reader and writer use a streaming API to avoid
30 * excessive memory use. The class can be used, e.g., as follows:
31 *
32 * <pre>
33 * DataTable dataTable = new ListDataTable("data", "dataTable", columns);
34 * Writer writer = new FileWriter("c:/data/data.xml");
35 * XMLData.writeData(writer, dataTable);
36 * </pre>
37 *
38 * The XML document has the following structure:
39 *
40 * <pre>
41 * <xmldata>
42 * <table id="tableId" description="description" class="org.djutils.data.ListDataTable">
43 * <column nr="0" id="obsNr" description="observation nr" type="int"></column>
44 * <column nr="1" id="value" description="observation value" type="double"></column>
45 * <column nr="2" id="comment" description="comment" type="java.lang.String"></column>
46 * </table>
47 * <data>
48 * <record index="0">
49 * <value nr="0" content="2"></value>
50 * <value nr="1" content="18.6"></value>
51 * <value nr="2" content="normal"></value>
52 * </record>
53 * <record index="1">
54 * <value nr="0" content="4"></value>
55 * <value nr="1" content="36.18"></value>
56 * <value nr="2" content="normal"></value>
57 * </record>
58 * </data>
59 * </xmldata>
60 * </pre>
61 *
62 * Copyright (c) 2020-2021 Delft University of Technology, Jaffalaan 5, 2628 BX Delft, the Netherlands. All rights reserved. See
63 * for project information <a href="https://djutils.org" target="_blank"> https://djutils.org</a>. The DJUTILS project is
64 * distributed under a three-clause BSD-style license, which can be found at
65 * <a href="https://djutils.org/docs/license.html" target="_blank"> https://djutils.org/docs/license.html</a>. <br>
66 * @author <a href="https://www.tudelft.nl/averbraeck">Alexander Verbraeck</a>
67 * @author <a href="https://www.tudelft.nl/pknoppers">Peter Knoppers</a>
68 * @author <a href="http://www.transport.citg.tudelft.nl">Wouter Schakel</a>
69 */
70 public final class XMLData
71 {
72 /**
73 * Utility class, no public constructor.
74 */
75 private XMLData()
76 {
77 // utility class
78 }
79
80 /**
81 * Write the data from the data table in XML format.
82 * @param writer Writer; the writer that writes the data, e.g. to a file
83 * @param dataTable DataTable; the data table to write
84 * @throws IOException on I/O error when writing the data
85 * @throws TextSerializationException on unknown data type for serialization
86 * @throws XMLStreamException on XML write error
87 */
88 public static void writeData(final Writer writer, final DataTable dataTable)
89 throws IOException, TextSerializationException, XMLStreamException
90 {
91 XMLStreamWriter xmlw = null;
92 try
93 {
94 XMLOutputFactory xmlOutputFactory = XMLOutputFactory.newInstance();
95 xmlw = xmlOutputFactory.createXMLStreamWriter(writer);
96
97 // XML header
98 xmlw.writeStartDocument();
99 xmlw.writeCharacters("\n");
100
101 // write the table metadata
102 xmlw.writeStartElement("xmldata");
103 xmlw.writeCharacters("\n");
104 xmlw.writeCharacters(" ");
105 xmlw.writeStartElement("table");
106 xmlw.writeAttribute("id", dataTable.getId());
107 xmlw.writeAttribute("description", dataTable.getDescription());
108 xmlw.writeAttribute("class", dataTable.getClass().getName());
109 xmlw.writeCharacters("\n");
110 int index = 0;
111 for (DataColumn<?> column : dataTable.getColumns())
112 {
113 xmlw.writeCharacters(" ");
114 xmlw.writeStartElement("column");
115 xmlw.writeAttribute("nr", String.valueOf(index++));
116 xmlw.writeAttribute("id", column.getId());
117 xmlw.writeAttribute("description", column.getDescription());
118 xmlw.writeAttribute("type", column.getValueType().getName());
119 xmlw.writeEndElement(); // column
120 xmlw.writeCharacters("\n");
121 }
122 xmlw.writeCharacters(" ");
123 xmlw.writeEndElement(); // table
124 xmlw.writeCharacters("\n");
125
126 // initialize the serializers
127 TextSerializer<?>[] serializers = new TextSerializer[dataTable.getNumberOfColumns()];
128 for (int i = 0; i < dataTable.getNumberOfColumns(); i++)
129 {
130 DataColumn<?> column = dataTable.getColumns().get(i);
131 serializers[i] = TextSerializer.resolve(column.getValueType());
132 }
133
134 // write the data
135 xmlw.writeCharacters(" ");
136 xmlw.writeStartElement("data");
137 xmlw.writeCharacters("\n");
138
139 // write the records
140 int recordNr = 0;
141 for (DataRecord record : dataTable)
142 {
143 Object[] values = record.getValues();
144 xmlw.writeCharacters(" ");
145 xmlw.writeStartElement("record");
146 xmlw.writeAttribute("index", String.valueOf(recordNr++));
147 xmlw.writeCharacters("\n");
148 for (int i = 0; i < dataTable.getNumberOfColumns(); i++)
149 {
150 xmlw.writeCharacters(" ");
151 xmlw.writeStartElement("value");
152 xmlw.writeAttribute("nr", String.valueOf(i));
153 xmlw.writeAttribute("content", serializers[i].serialize(values[i]));
154 xmlw.writeEndElement(); // value
155 xmlw.writeCharacters("\n");
156 }
157 xmlw.writeCharacters(" ");
158 xmlw.writeEndElement(); // record
159 xmlw.writeCharacters("\n");
160 }
161
162 // end XML document
163 xmlw.writeCharacters(" ");
164 xmlw.writeEndElement(); // data
165 xmlw.writeCharacters("\n");
166 xmlw.writeEndElement(); // xmldata
167 xmlw.writeCharacters("\n");
168 xmlw.writeEndDocument();
169 }
170 finally
171 {
172 if (null != xmlw)
173 {
174 xmlw.close();
175 }
176 }
177 }
178
179 /**
180 * Write the data from the data table in XML format.
181 * @param filename String; the file name to write the data to
182 * @param dataTable DataTable; the data table to write
183 * @throws IOException on I/O error when writing the data
184 * @throws TextSerializationException on unknown data type for serialization
185 * @throws XMLStreamException on XML write error
186 */
187 public static void writeData(final String filename, final DataTable dataTable)
188 throws IOException, TextSerializationException, XMLStreamException
189 {
190 FileWriter fw = null;
191 try
192 {
193 fw = new FileWriter(filename);
194 writeData(fw, dataTable);
195 }
196 finally
197 {
198 if (null != fw)
199 {
200 fw.close();
201 }
202 }
203 }
204
205 /**
206 * Read the data from the XML-file into the data table. Use the metadata to reconstruct the data table.
207 * @param reader Reader; the reader that can read the data, e.g. from a file
208 * @return dataTable the data table reconstructed from the meta data and filled with the data
209 * @throws IOException on I/O error when reading the data
210 * @throws TextSerializationException on unknown data type for serialization
211 * @throws XMLStreamException on XML read error
212 */
213 public static DataTable readData(final Reader reader) throws IOException, TextSerializationException, XMLStreamException
214 {
215 XMLStreamReader xmlr = null;
216 try
217 {
218 // read the metadata file and reconstruct the data table
219 XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance();
220 xmlr = xmlInputFactory.createXMLStreamReader(reader);
221
222 // wait for the xmldata tag
223 waitFor(xmlr, "xmldata");
224
225 // wait for the table tag
226 waitFor(xmlr, "table");
227 String[] tableProperties = getAttributes(xmlr, "id", "description", "class");
228 Throw.when(!tableProperties[2].endsWith("ListDataTable"), IOException.class,
229 "Currently, this method can only recreate a ListDataTable");
230
231 // column metadata
232 List<DataColumn<?>> columns = new ArrayList<>();
233 int index = 0;
234 while (waitFor(xmlr, "column", "table"))
235 {
236 String[] columnProperties = getAttributes(xmlr, "nr", "id", "description", "type");
237 if (Integer.valueOf(columnProperties[0]).intValue() != index)
238 {
239 throw new IOException("column nr not ok");
240 }
241 String type = columnProperties[3];
242 Class<?> valueClass = Primitive.forName(type);
243 if (valueClass == null)
244 {
245 try
246 {
247 valueClass = Class.forName(type);
248 }
249 catch (ClassNotFoundException exception)
250 {
251 throw new IOException("Could not find class " + type, exception);
252 }
253 }
254 @SuppressWarnings({"rawtypes", "unchecked"})
255 DataColumn<?> column = new SimpleDataColumn(columnProperties[1], columnProperties[2], valueClass);
256 columns.add(column);
257 index++;
258 }
259 ListDataTablestDataTable">ListDataTable dataTable = new ListDataTable(tableProperties[0], tableProperties[1], columns);
260
261 // obtain the serializers
262 TextSerializer<?>[] serializers = new TextSerializer[dataTable.getNumberOfColumns()];
263 for (int i = 0; i < dataTable.getNumberOfColumns(); i++)
264 {
265 DataColumn<?> column = dataTable.getColumns().get(i);
266 serializers[i] = TextSerializer.resolve(column.getValueType());
267 }
268
269 // read the data file records
270 waitFor(xmlr, "data");
271 while (waitFor(xmlr, "record", "data"))
272 {
273 String[] data = new String[columns.size()];
274 while (waitFor(xmlr, "value", "record"))
275 {
276 String[] valueProperties = getAttributes(xmlr, "nr", "content");
277 data[Integer.valueOf(valueProperties[0]).intValue()] = valueProperties[1];
278 }
279 Object[] values = new Object[columns.size()];
280 for (int i = 0; i < values.length; i++)
281 {
282 values[i] = serializers[i].deserialize(data[i]);
283 }
284 dataTable.addRecord(values);
285 }
286 return dataTable;
287 }
288 finally
289 {
290 if (null != xmlr)
291 {
292 xmlr.close();
293 }
294 }
295 }
296
297 /**
298 * Read from the XML file until a START_ELEMENT with the id equal to the provided tag is encountered.
299 * @param xmlr XMLStreamReader; the XML stream reader
300 * @param tag String; the tag to retrieve
301 * @throws XMLStreamException on error reading from the XML stream
302 * @throws IOException when the stream ended without finding the tag
303 */
304 private static void waitFor(final XMLStreamReader xmlr, final String tag) throws XMLStreamException, IOException
305 {
306 while (xmlr.hasNext())
307 {
308 xmlr.next();
309 if (xmlr.getEventType() == XMLStreamConstants.START_ELEMENT)
310 {
311 if (xmlr.getLocalName().equals(tag))
312 {
313 return;
314 }
315 }
316 }
317 throw new IOException("Unexpected end of stream");
318 }
319
320 /**
321 * Read from the XML file until a START_ELEMENT with the id equal to the provided tag is encountered, or until the
322 * stopEndTag is reached. This can be used to get the starting tag in a repeat group. When the starting tag is found, the
323 * method returns true. When the end tag of the repeat group is found, false is returned.
324 * @param xmlr XMLStreamReader; the XML stream reader
325 * @param tag String; the tag to retrieve, usually a tag in a repeat group
326 * @param stopEndTag String; the tag to indicate the end of the repeat group
327 * @return true when the tag in the repeat group was found; false when the stop tag was found
328 * @throws XMLStreamException on error reading from the XML stream
329 * @throws IOException when the stream ended without finding the tag or the stop tag
330 */
331 private static boolean waitFor(final XMLStreamReader xmlr, final String tag, final String stopEndTag)
332 throws XMLStreamException, IOException
333 {
334 while (xmlr.hasNext())
335 {
336 xmlr.next();
337 if (xmlr.getEventType() == XMLStreamConstants.START_ELEMENT)
338 {
339 if (xmlr.getLocalName().equals(tag))
340 {
341 return true;
342 }
343 }
344 else if (xmlr.getEventType() == XMLStreamConstants.END_ELEMENT)
345 {
346 if (xmlr.getLocalName().equals(stopEndTag))
347 {
348 return false;
349 }
350 }
351 }
352 throw new IOException("Unexpected end of stream");
353 }
354
355 /**
356 * Read the attributes into an array and return the array. The position of each attribute is indicated by the vararg
357 * parameter 'attributes'.
358 * @param xmlr XMLStreamReader; the XML stream reader
359 * @param attributes String...; the attributes that are expected
360 * @return the array of atribute values, in the order of the vararg parameter 'attributes'
361 * @throws XMLStreamException on error reading from the XML stream
362 * @throws IOException when the current element does not contain the right (number of) attributes
363 */
364 private static String[] getAttributes(final XMLStreamReader xmlr, final String... attributes)
365 throws XMLStreamException, IOException
366 {
367 String[] result = new String[attributes.length];
368 int found = 0;
369 for (int i = 0; i < xmlr.getAttributeCount(); i++)
370 {
371 String localName = xmlr.getAttributeLocalName(i);
372 String value = xmlr.getAttributeValue(i);
373 for (int j = 0; j < attributes.length; j++)
374 {
375 if (localName.equals(attributes[j]))
376 {
377 result[j] = value;
378 found++;
379 }
380 }
381 }
382 Throw.when(found != attributes.length, IOException.class, "attribute data does not contain %d fields",
383 attributes.length);
384 return result;
385 }
386
387 /**
388 * Read the data from the XML-file into the data table. Use the metadata to reconstruct the data table.
389 * @param filename String; the file name to read the data from
390 * @return dataTable the data table reconstructed from the meta data and filled with the data
391 * @throws IOException on I/O error when reading the data
392 * @throws TextSerializationException on unknown data type for serialization
393 * @throws XMLStreamException on XML read error
394 */
395 public static DataTable readData(final String filename) throws IOException, TextSerializationException, XMLStreamException
396 {
397 FileReader fr = null;
398 try
399 {
400 fr = new FileReader(filename);
401 return readData(fr);
402 }
403 finally
404 {
405 if (null != fr)
406 {
407 fr.close();
408 }
409 }
410 }
411
412 }