1 package org.djutils.data.xml;
2
3 import java.io.FileReader;
4 import java.io.FileWriter;
5 import java.io.IOException;
6 import java.io.Reader;
7 import java.io.Writer;
8 import java.util.ArrayList;
9 import java.util.List;
10
11 import javax.xml.stream.XMLInputFactory;
12 import javax.xml.stream.XMLOutputFactory;
13 import javax.xml.stream.XMLStreamConstants;
14 import javax.xml.stream.XMLStreamException;
15 import javax.xml.stream.XMLStreamReader;
16 import javax.xml.stream.XMLStreamWriter;
17
18 import org.djutils.data.DataColumn;
19 import org.djutils.data.DataRecord;
20 import org.djutils.data.DataTable;
21 import org.djutils.data.ListDataTable;
22 import org.djutils.data.SimpleDataColumn;
23 import org.djutils.data.serialization.TextSerializationException;
24 import org.djutils.data.serialization.TextSerializer;
25 import org.djutils.exceptions.Throw;
26 import org.djutils.primitives.Primitive;
27
28 /**
29 * XMLData takes care of reading and writing of table data in XML format. The reader and writer use a streaming API to avoid
30 * excessive memory use. The class can be used, e.g., as follows:
31 *
32 * <pre>
33 * DataTable dataTable = new ListDataTable("data", "dataTable", columns);
34 * Writer writer = new FileWriter("c:/data/data.xml");
35 * XMLData.writeData(writer, dataTable);
36 * </pre>
37 *
38 * The XML document has the following structure:
39 *
40 * <pre>
41 * <xmldata>
42 * <table id="tableId" description="description" class="org.djutils.data.ListDataTable">
43 * <column nr="0" id="obsNr" description="observation nr" type="int"></column>
44 * <column nr="1" id="value" description="observation value" type="double"></column>
45 * <column nr="2" id="comment" description="comment" type="java.lang.String"></column>
46 * </table>
47 * <data>
48 * <record index="0">
49 * <value nr="0" content="2"></value>
50 * <value nr="1" content="18.6"></value>
51 * <value nr="2" content="normal"></value>
52 * </record>
53 * <record index="1">
54 * <value nr="0" content="4"></value>
55 * <value nr="1" content="36.18"></value>
56 * <value nr="2" content="normal"></value>
57 * </record>
58 * </data>
59 * </xmldata>
60 * </pre>
61 *
62 * Copyright (c) 2020-2020 Delft University of Technology, Jaffalaan 5, 2628 BX Delft, the Netherlands. All rights reserved. See
63 * for project information <a href="https://djutils.org" target="_blank"> https://djutils.org</a>. The DJUTILS project is
64 * distributed under a three-clause BSD-style license, which can be found at
65 * <a href="https://djutils.org/docs/license.html" target="_blank"> https://djutils.org/docs/license.html</a>. <br>
66 * @author <a href="https://www.tudelft.nl/averbraeck">Alexander Verbraeck</a>
67 * @author <a href="https://www.tudelft.nl/pknoppers">Peter Knoppers</a>
68 * @author <a href="http://www.transport.citg.tudelft.nl">Wouter Schakel</a>
69 */
70 public final class XMLData
71 {
72 /**
73 * Utility class, no public constructor.
74 */
75 private XMLData()
76 {
77 // utility class
78 }
79
80 /**
81 * Write the data from the data table in XML format.
82 * @param writer Writer; the writer that writes the data, e.g. to a file
83 * @param dataTable DataTable; the data table to write
84
85 * @throws IOException on I/O error when writing the data
86 * @throws TextSerializationException on unknown data type for serialization
87 * @throws XMLStreamException on XML write error
88 */
89 public static void writeData(final Writer writer, final DataTable dataTable)
90 throws IOException, TextSerializationException, XMLStreamException
91 {
92 XMLStreamWriter xmlw = null;
93 try
94 {
95 XMLOutputFactory xmlOutputFactory = XMLOutputFactory.newInstance();
96 xmlw = xmlOutputFactory.createXMLStreamWriter(writer);
97
98 // XML header
99 xmlw.writeStartDocument();
100 xmlw.writeCharacters("\n");
101
102 // write the table metadata
103 xmlw.writeStartElement("xmldata");
104 xmlw.writeCharacters("\n");
105 xmlw.writeCharacters(" ");
106 xmlw.writeStartElement("table");
107 xmlw.writeAttribute("id", dataTable.getId());
108 xmlw.writeAttribute("description", dataTable.getDescription());
109 xmlw.writeAttribute("class", dataTable.getClass().getName());
110 xmlw.writeCharacters("\n");
111 int index = 0;
112 for (DataColumn<?> column : dataTable.getColumns())
113 {
114 xmlw.writeCharacters(" ");
115 xmlw.writeStartElement("column");
116 xmlw.writeAttribute("nr", String.valueOf(index++));
117 xmlw.writeAttribute("id", column.getId());
118 xmlw.writeAttribute("description", column.getDescription());
119 xmlw.writeAttribute("type", column.getValueType().getName());
120 xmlw.writeEndElement(); // column
121 xmlw.writeCharacters("\n");
122 }
123 xmlw.writeCharacters(" ");
124 xmlw.writeEndElement(); // table
125 xmlw.writeCharacters("\n");
126
127 // initialize the serializers
128 TextSerializer<?>[] serializers = new TextSerializer[dataTable.getNumberOfColumns()];
129 for (int i = 0; i < dataTable.getNumberOfColumns(); i++)
130 {
131 DataColumn<?> column = dataTable.getColumns().get(i);
132 serializers[i] = TextSerializer.resolve(column.getValueType());
133 }
134
135 // write the data
136 xmlw.writeCharacters(" ");
137 xmlw.writeStartElement("data");
138 xmlw.writeCharacters("\n");
139
140 // write the records
141 int recordNr = 0;
142 for (DataRecord record : dataTable)
143 {
144 Object[] values = record.getValues();
145 xmlw.writeCharacters(" ");
146 xmlw.writeStartElement("record");
147 xmlw.writeAttribute("index", String.valueOf(recordNr++));
148 xmlw.writeCharacters("\n");
149 for (int i = 0; i < dataTable.getNumberOfColumns(); i++)
150 {
151 xmlw.writeCharacters(" ");
152 xmlw.writeStartElement("value");
153 xmlw.writeAttribute("nr", String.valueOf(i));
154 xmlw.writeAttribute("content", serializers[i].serialize(values[i]));
155 xmlw.writeEndElement(); // value
156 xmlw.writeCharacters("\n");
157 }
158 xmlw.writeCharacters(" ");
159 xmlw.writeEndElement(); // record
160 xmlw.writeCharacters("\n");
161 }
162
163 // end XML document
164 xmlw.writeCharacters(" ");
165 xmlw.writeEndElement(); // data
166 xmlw.writeCharacters("\n");
167 xmlw.writeEndElement(); // xmldata
168 xmlw.writeCharacters("\n");
169 xmlw.writeEndDocument();
170 }
171 finally
172 {
173 if (null != xmlw)
174 {
175 xmlw.close();
176 }
177 }
178 }
179
180 /**
181 * Write the data from the data table in XML format.
182 * @param filename String; the file name to write the data to
183 * @param dataTable DataTable; the data table to write
184
185 * @throws IOException on I/O error when writing the data
186 * @throws TextSerializationException on unknown data type for serialization
187 * @throws XMLStreamException on XML write error
188 */
189 public static void writeData(final String filename, final DataTable dataTable)
190 throws IOException, TextSerializationException, XMLStreamException
191 {
192 FileWriter fw = null;
193 try
194 {
195 fw = new FileWriter(filename);
196 writeData(fw, dataTable);
197 }
198 finally
199 {
200 if (null != fw)
201 {
202 fw.close();
203 }
204 }
205 }
206
207 /**
208 * Read the data from the XML-file into the data table. Use the metadata to reconstruct the data table.
209 * @param reader Reader; the reader that can read the data, e.g. from a file
210 * @return dataTable the data table reconstructed from the meta data and filled with the data
211 * @throws IOException on I/O error when reading the data
212 * @throws TextSerializationException on unknown data type for serialization
213 * @throws XMLStreamException on XML read error
214 */
215 public static DataTable readData(final Reader reader) throws IOException, TextSerializationException, XMLStreamException
216 {
217 XMLStreamReader xmlr = null;
218 try
219 {
220 // read the metadata file and reconstruct the data table
221 XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance();
222 xmlr = xmlInputFactory.createXMLStreamReader(reader);
223
224 // wait for the xmldata tag
225 waitFor(xmlr, "xmldata");
226
227 // wait for the table tag
228 waitFor(xmlr, "table");
229 String[] tableProperties = getAttributes(xmlr, "id", "description", "class");
230 Throw.when(!tableProperties[2].endsWith("ListDataTable"), IOException.class,
231 "Currently, this method can only recreate a ListDataTable");
232
233 // column metadata
234 List<DataColumn<?>> columns = new ArrayList<>();
235 int index = 0;
236 while (waitFor(xmlr, "column", "table"))
237 {
238 String[] columnProperties = getAttributes(xmlr, "nr", "id", "description", "type");
239 if (Integer.valueOf(columnProperties[0]).intValue() != index)
240 {
241 throw new IOException("column nr not ok");
242 }
243 String type = columnProperties[3];
244 Class<?> valueClass = Primitive.forName(type);
245 if (valueClass == null)
246 {
247 try
248 {
249 valueClass = Class.forName(type);
250 }
251 catch (ClassNotFoundException exception)
252 {
253 throw new IOException("Could not find class " + type, exception);
254 }
255 }
256 @SuppressWarnings({ "rawtypes", "unchecked" })
257 DataColumn<?> column = new SimpleDataColumn(columnProperties[1], columnProperties[2], valueClass);
258 columns.add(column);
259 index++;
260 }
261 ListDataTablestDataTable">ListDataTable dataTable = new ListDataTable(tableProperties[0], tableProperties[1], columns);
262
263 // obtain the serializers
264 TextSerializer<?>[] serializers = new TextSerializer[dataTable.getNumberOfColumns()];
265 for (int i = 0; i < dataTable.getNumberOfColumns(); i++)
266 {
267 DataColumn<?> column = dataTable.getColumns().get(i);
268 serializers[i] = TextSerializer.resolve(column.getValueType());
269 }
270
271 // read the data file records
272 waitFor(xmlr, "data");
273 while (waitFor(xmlr, "record", "data"))
274 {
275 String[] data = new String[columns.size()];
276 while (waitFor(xmlr, "value", "record"))
277 {
278 String[] valueProperties = getAttributes(xmlr, "nr", "content");
279 data[Integer.valueOf(valueProperties[0]).intValue()] = valueProperties[1];
280 }
281 Object[] values = new Object[columns.size()];
282 for (int i = 0; i < values.length; i++)
283 {
284 values[i] = serializers[i].deserialize(data[i]);
285 }
286 dataTable.addRecord(values);
287 }
288 return dataTable;
289 }
290 finally
291 {
292 if (null != xmlr)
293 {
294 xmlr.close();
295 }
296 }
297 }
298
299 /**
300 * Read from the XML file until a START_ELEMENT with the id equal to the provided tag is encountered.
301 * @param xmlr XMLStreamReader; the XML stream reader
302
303 * @param tag String; the tag to retrieve
304
305 * @throws XMLStreamException on error reading from the XML stream
306 * @throws IOException when the stream ended without finding the tag
307 */
308 private static void waitFor(final XMLStreamReader xmlr, final String tag) throws XMLStreamException, IOException
309 {
310 while (xmlr.hasNext())
311 {
312 xmlr.next();
313 if (xmlr.getEventType() == XMLStreamConstants.START_ELEMENT)
314 {
315 if (xmlr.getLocalName().equals(tag))
316 {
317 return;
318 }
319 }
320 }
321 throw new IOException("Unexpected end of stream");
322 }
323
324 /**
325 * Read from the XML file until a START_ELEMENT with the id equal to the provided tag is encountered, or until the
326 * stopEndTag is reached. This can be used to get the starting tag in a repeat group. When the starting tag is found, the
327 * method returns true. When the end tag of the repeat group is found, false is returned.
328 * @param xmlr XMLStreamReader; the XML stream reader
329
330 * @param tag String; the tag to retrieve, usually a tag in a repeat group
331
332 * @param stopEndTag String; the tag to indicate the end of the repeat group
333
334 * @return true when the tag in the repeat group was found; false when the stop tag was found
335 * @throws XMLStreamException on error reading from the XML stream
336 * @throws IOException when the stream ended without finding the tag or the stop tag
337 */
338 private static boolean waitFor(final XMLStreamReader xmlr, final String tag, final String stopEndTag)
339 throws XMLStreamException, IOException
340 {
341 while (xmlr.hasNext())
342 {
343 xmlr.next();
344 if (xmlr.getEventType() == XMLStreamConstants.START_ELEMENT)
345 {
346 if (xmlr.getLocalName().equals(tag))
347 {
348 return true;
349 }
350 }
351 else if (xmlr.getEventType() == XMLStreamConstants.END_ELEMENT)
352 {
353 if (xmlr.getLocalName().equals(stopEndTag))
354 {
355 return false;
356 }
357 }
358 }
359 throw new IOException("Unexpected end of stream");
360 }
361
362 /**
363 * Read the attributes into an array and return the array. The position of each attribute is indicated by the vararg
364 * parameter 'attributes'.
365 * @param xmlr XMLStreamReader; the XML stream reader
366
367 * @param attributes String...; the attributes that are expected
368
369 * @return the array of atribute values, in the order of the vararg parameter 'attributes'
370 * @throws XMLStreamException on error reading from the XML stream
371 * @throws IOException when the current element does not contain the right (number of) attributes
372 */
373 private static String[] getAttributes(final XMLStreamReader xmlr, final String... attributes)
374 throws XMLStreamException, IOException
375 {
376 String[] result = new String[attributes.length];
377 int found = 0;
378 for (int i = 0; i < xmlr.getAttributeCount(); i++)
379 {
380 String localName = xmlr.getAttributeLocalName(i);
381 String value = xmlr.getAttributeValue(i);
382 for (int j = 0; j < attributes.length; j++)
383 {
384 if (localName.equals(attributes[j]))
385 {
386 result[j] = value;
387 found++;
388 }
389 }
390 }
391 Throw.when(found != attributes.length, IOException.class, "attribute data does not contain %d fields",
392 attributes.length);
393 return result;
394 }
395
396 /**
397 * Read the data from the XML-file into the data table. Use the metadata to reconstruct the data table.
398 * @param filename String; the file name to read the data from
399 * @return dataTable the data table reconstructed from the meta data and filled with the data
400 * @throws IOException on I/O error when reading the data
401 * @throws TextSerializationException on unknown data type for serialization
402 * @throws XMLStreamException on XML read error
403 */
404 public static DataTable readData(final String filename) throws IOException, TextSerializationException, XMLStreamException
405 {
406 FileReader fr = null;
407 try
408 {
409 fr = new FileReader(filename);
410 return readData(fr);
411 }
412 finally
413 {
414 if (null != fr)
415 {
416 fr.close();
417 }
418 }
419 }
420
421 }