View Javadoc
1   package org.djutils.data.xml;
2   
3   import java.io.FileReader;
4   import java.io.FileWriter;
5   import java.io.IOException;
6   import java.io.Reader;
7   import java.io.Writer;
8   import java.util.ArrayList;
9   import java.util.List;
10  import java.util.function.Consumer;
11  
12  import javax.xml.stream.XMLInputFactory;
13  import javax.xml.stream.XMLOutputFactory;
14  import javax.xml.stream.XMLStreamConstants;
15  import javax.xml.stream.XMLStreamException;
16  import javax.xml.stream.XMLStreamReader;
17  import javax.xml.stream.XMLStreamWriter;
18  
19  import org.djutils.data.Column;
20  import org.djutils.data.ListTable;
21  import org.djutils.data.Row;
22  import org.djutils.data.Table;
23  import org.djutils.data.serialization.TextSerializationException;
24  import org.djutils.data.serialization.TextSerializer;
25  import org.djutils.exceptions.Throw;
26  import org.djutils.primitives.Primitive;
27  
28  /**
29   * XmlData takes care of reading and writing of table data in XML format. The reader and writer use a streaming API to avoid
30   * excessive memory use. The class can be used, e.g., as follows:
31   * 
32   * <pre>
33   * Table dataTable = new ListTable("data", "dataTable", columns);
34   * Writer writer = new FileWriter("c:/data/data.xml");
35   * XmlData.writeData(writer, dataTable);
36   * </pre>
37   * 
38   * The XML document has the following structure:
39   * 
40   * <pre>
41   * &lt;xmldata&gt;
42   * &nbsp;&nbsp;&lt;table id="tableId" description="description" class="org.djutils.data.ListTable"&gt;
43   * &nbsp;&nbsp;&nbsp;&nbsp;&lt;column nr="0" id="obsNr" description="observation nr" type="int"&gt;&lt;/column&gt;
44   * &nbsp;&nbsp;&nbsp;&nbsp;&lt;column nr="1" id="value" description="observation value" type="double"&gt;&lt;/column&gt;
45   * &nbsp;&nbsp;&nbsp;&nbsp;&lt;column nr="2" id="comment" description="comment" type="java.lang.String"&gt;&lt;/column&gt;
46   * &nbsp;&nbsp;&lt;/table&gt;
47   * &nbsp;&nbsp;&lt;data&gt;
48   * &nbsp;&nbsp;&nbsp;&nbsp;&lt;record index="0"&gt;
49   * &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&lt;value nr="0" content="2"&gt;&lt;/value&gt;
50   * &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&lt;value nr="1" content="18.6"&gt;&lt;/value&gt;
51   * &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&lt;value nr="2" content="normal"&gt;&lt;/value&gt;
52   * &nbsp;&nbsp;&nbsp;&nbsp;&lt;/record&gt;
53   * &nbsp;&nbsp;&nbsp;&nbsp;&lt;record index="1"&gt;
54   * &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&lt;value nr="0" content="4"&gt;&lt;/value&gt;
55   * &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&lt;value nr="1" content="36.18"&gt;&lt;/value&gt;
56   * &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&lt;value nr="2" content="normal"&gt;&lt;/value&gt;
57   * &nbsp;&nbsp;&nbsp;&nbsp;&lt;/record&gt;
58   * &nbsp;&nbsp;&lt;/data&gt;
59   * &lt;/xmldata&gt;
60   * </pre>
61   * <p>
62   * Copyright (c) 2020-2024 Delft University of Technology, PO Box 5, 2600 AA, Delft, the Netherlands. All rights reserved. <br>
63   * BSD-style license. See <a href="https://djutils.org/docs/current/djutils/licenses.html">DJUTILS License</a>.
64   * </p>
65   * @author <a href="https://github.com/averbraeck">Alexander Verbraeck</a>
66   * @author <a href="https://tudelft.nl/staff/p.knoppers-1">Peter Knoppers</a>
67   * @author <a href="https://dittlab.tudelft.nl">Wouter Schakel</a>
68   */
69  public final class XmlData
70  {
71      /**
72       * Utility class, no public constructor.
73       */
74      private XmlData()
75      {
76          // utility class
77      }
78  
79      /**
80       * Write the data from the data table in XML format.
81       * @param writer Writer; the writer that writes the data, e.g. to a file
82       * @param dataTable Table; the data table to write
83       * @throws IOException on I/O error when writing the data
84       * @throws TextSerializationException on unknown data type for serialization
85       * @throws XMLStreamException on XML write error
86       */
87      public static void writeData(final Writer writer, final Table dataTable)
88              throws IOException, TextSerializationException, XMLStreamException
89      {
90          XMLStreamWriter xmlw = null;
91          try
92          {
93              XMLOutputFactory xmlOutputFactory = XMLOutputFactory.newInstance();
94              xmlw = xmlOutputFactory.createXMLStreamWriter(writer);
95  
96              // XML header
97              xmlw.writeStartDocument();
98              xmlw.writeCharacters("\n");
99  
100             // write the table metadata
101             xmlw.writeStartElement("xmldata");
102             xmlw.writeCharacters("\n");
103             xmlw.writeCharacters("  ");
104             xmlw.writeStartElement("table");
105             xmlw.writeAttribute("id", dataTable.getId());
106             xmlw.writeAttribute("description", dataTable.getDescription());
107             xmlw.writeAttribute("class", dataTable.getClass().getName());
108             xmlw.writeCharacters("\n");
109             int index = 0;
110             for (Column<?> column : dataTable.getColumns())
111             {
112                 xmlw.writeCharacters("    ");
113                 xmlw.writeEmptyElement("column");
114                 xmlw.writeAttribute("nr", String.valueOf(index++));
115                 xmlw.writeAttribute("id", column.getId());
116                 xmlw.writeAttribute("description", column.getDescription());
117                 xmlw.writeAttribute("type", column.getValueType().getName());
118                 if (column.getUnit() != null)
119                 {
120                     xmlw.writeAttribute("unit", column.getUnit());
121                 }
122                 xmlw.writeCharacters("\n");
123             }
124             xmlw.writeCharacters("  ");
125             xmlw.writeEndElement(); // table
126             xmlw.writeCharacters("\n");
127 
128             // initialize the serializers
129             TextSerializer<?>[] serializers = new TextSerializer[dataTable.getNumberOfColumns()];
130             for (int i = 0; i < dataTable.getNumberOfColumns(); i++)
131             {
132                 Column<?> column = dataTable.getColumns().get(i);
133                 serializers[i] = TextSerializer.resolve(column.getValueType());
134             }
135 
136             // write the data
137             xmlw.writeCharacters("  ");
138             xmlw.writeStartElement("data");
139             xmlw.writeCharacters("\n");
140 
141             // write the records
142             int recordNr = 0;
143             for (Row row : dataTable)
144             {
145                 Object[] values = row.getValues();
146                 xmlw.writeCharacters("    ");
147                 xmlw.writeStartElement("row");
148                 xmlw.writeAttribute("index", String.valueOf(recordNr++));
149                 xmlw.writeCharacters("\n");
150                 for (int i = 0; i < dataTable.getNumberOfColumns(); i++)
151                 {
152                     xmlw.writeCharacters("      ");
153                     xmlw.writeEmptyElement("value");
154                     xmlw.writeAttribute("nr", String.valueOf(i));
155                     String content = TextSerializer.serialize(serializers[i], values[i], dataTable.getColumn(i).getUnit());
156                     if (content != null)
157                     {
158                         xmlw.writeAttribute("content", content);
159                     }
160                     xmlw.writeCharacters("\n");
161                 }
162                 xmlw.writeCharacters("    ");
163                 xmlw.writeEndElement(); // row
164                 xmlw.writeCharacters("\n");
165             }
166 
167             // end XML document
168             xmlw.writeCharacters("  ");
169             xmlw.writeEndElement(); // data
170             xmlw.writeCharacters("\n");
171             xmlw.writeEndElement(); // xmldata
172             xmlw.writeCharacters("\n");
173             xmlw.writeEndDocument();
174         }
175         finally
176         {
177             if (null != xmlw)
178             {
179                 xmlw.close();
180             }
181         }
182     }
183 
184     /**
185      * Write the data from the data table in XML format.
186      * @param filename String; the file name to write the data to
187      * @param dataTable Table; the data table to write
188      * @throws IOException on I/O error when writing the data
189      * @throws TextSerializationException on unknown data type for serialization
190      * @throws XMLStreamException on XML write error
191      */
192     public static void writeData(final String filename, final Table dataTable)
193             throws IOException, TextSerializationException, XMLStreamException
194     {
195         try (FileWriter fw = new FileWriter(filename))
196         {
197             writeData(fw, dataTable);
198         }
199     }
200 
201     /**
202      * Read the data from the XML-file into the data table. Use the metadata to reconstruct the data table.
203      * @param reader Reader; the reader that can read the data, e.g. from a file
204      * @return dataTable the data table reconstructed from the meta data and filled with the data
205      * @throws IOException on I/O error when reading the data
206      * @throws TextSerializationException on unknown data type for serialization
207      * @throws XMLStreamException on XML read error
208      */
209     public static Table readData(final Reader reader) throws IOException, TextSerializationException, XMLStreamException
210     {
211         XMLStreamReader xmlr = null;
212         try
213         {
214             // read the metadata file and reconstruct the data table
215             XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance();
216             xmlr = xmlInputFactory.createXMLStreamReader(reader);
217 
218             // wait for the xmldata tag
219             waitFor(xmlr, "xmldata");
220 
221             // wait for the table tag
222             waitFor(xmlr, "table");
223             String[] tableProperties = getAttributes(xmlr, "id", "description", "class");
224 
225             // column metadata
226             List<Column<?>> columns = new ArrayList<>();
227             int index = 0;
228             while (waitFor(xmlr, "column", "table"))
229             {
230                 String[] columnProperties;
231                 if (xmlr.getAttributeCount() == 5)
232                 {
233                     columnProperties = getAttributes(xmlr, "nr", "id", "description", "type", "unit");
234                 }
235                 else
236                 {
237                     // null unit
238                     columnProperties = getAttributes(xmlr, "nr", "id", "description", "type");
239                 }
240                 if (Integer.valueOf(columnProperties[0]).intValue() != index)
241                 {
242                     throw new IOException("column nr not ok");
243                 }
244                 String type = columnProperties[3];
245                 Class<?> valueClass = Primitive.forName(type);
246                 if (valueClass == null)
247                 {
248                     try
249                     {
250                         valueClass = Class.forName(type);
251                     }
252                     catch (ClassNotFoundException exception)
253                     {
254                         throw new IOException("Could not find class " + type, exception);
255                     }
256                 }
257                 Column<?> column = new Column<>(columnProperties[1], columnProperties[2], valueClass,
258                         columnProperties.length >= 5 ? columnProperties[4] : null);
259                 columns.add(column);
260                 index++;
261             }
262 
263             // create table
264             Table table;
265             Consumer<Object[]> unserializableTable;
266             if (tableProperties[2].equals(ListTable.class.getName()))
267             {
268                 ListTable listTable = new ListTable(tableProperties[0], tableProperties[1], columns);
269                 table = listTable;
270                 unserializableTable = (
271                         data
272                 ) -> listTable.addRow(data);
273             }
274             else
275             {
276                 // fallback
277                 ListTable listTable = new ListTable(tableProperties[0], tableProperties[1], columns);
278                 table = listTable;
279                 unserializableTable = (
280                         data
281                 ) -> listTable.addRow(data);
282             }
283 
284             // obtain the serializers
285             TextSerializer<?>[] serializers = new TextSerializer[table.getNumberOfColumns()];
286             for (int i = 0; i < table.getNumberOfColumns(); i++)
287             {
288                 serializers[i] = TextSerializer.resolve(columns.get(i).getValueType());
289             }
290 
291             // read the data file records
292             waitFor(xmlr, "data");
293             while (waitFor(xmlr, "row", "data"))
294             {
295                 String[] data = new String[columns.size()];
296                 while (waitFor(xmlr, "value", "row"))
297                 {
298                     if (xmlr.getAttributeCount() == 2)
299                     {
300                         String[] valueProperties = getAttributes(xmlr, "nr", "content");
301                         data[Integer.valueOf(valueProperties[0]).intValue()] = valueProperties[1];
302                     }
303                     else
304                     {
305                         // null content
306                         String[] valueProperties = getAttributes(xmlr, "nr");
307                         data[Integer.valueOf(valueProperties[0]).intValue()] = null;
308                     }
309                 }
310                 Object[] values = new Object[columns.size()];
311                 for (int i = 0; i < values.length; i++)
312                 {
313                     values[i] = TextSerializer.deserialize(serializers[i], data[i], columns.get(i));
314                 }
315                 unserializableTable.accept(values); // addRow
316             }
317             return table;
318         }
319         finally
320         {
321             if (null != xmlr)
322             {
323                 xmlr.close();
324             }
325         }
326     }
327 
328     /**
329      * Read from the XML file until a START_ELEMENT with the id equal to the provided tag is encountered.
330      * @param xmlr XMLStreamReader; the XML stream reader
331      * @param tag String; the tag to retrieve
332      * @throws XMLStreamException on error reading from the XML stream
333      * @throws IOException when the stream ended without finding the tag
334      */
335     private static void waitFor(final XMLStreamReader xmlr, final String tag) throws XMLStreamException, IOException
336     {
337         while (xmlr.hasNext())
338         {
339             xmlr.next();
340             if (xmlr.getEventType() == XMLStreamConstants.START_ELEMENT)
341             {
342                 if (xmlr.getLocalName().equals(tag))
343                 {
344                     return;
345                 }
346             }
347         }
348         throw new IOException("Unexpected end of stream");
349     }
350 
351     /**
352      * Read from the XML file until a START_ELEMENT with the id equal to the provided tag is encountered, or until the
353      * stopEndTag is reached. This can be used to get the starting tag in a repeat group. When the starting tag is found, the
354      * method returns true. When the end tag of the repeat group is found, false is returned.
355      * @param xmlr XMLStreamReader; the XML stream reader
356      * @param tag String; the tag to retrieve, usually a tag in a repeat group
357      * @param stopEndTag String; the tag to indicate the end of the repeat group
358      * @return true when the tag in the repeat group was found; false when the stop tag was found
359      * @throws XMLStreamException on error reading from the XML stream
360      * @throws IOException when the stream ended without finding the tag or the stop tag
361      */
362     private static boolean waitFor(final XMLStreamReader xmlr, final String tag, final String stopEndTag)
363             throws XMLStreamException, IOException
364     {
365         while (xmlr.hasNext())
366         {
367             xmlr.next();
368             if (xmlr.getEventType() == XMLStreamConstants.START_ELEMENT)
369             {
370                 if (xmlr.getLocalName().equals(tag))
371                 {
372                     return true;
373                 }
374             }
375             else if (xmlr.getEventType() == XMLStreamConstants.END_ELEMENT)
376             {
377                 if (xmlr.getLocalName().equals(stopEndTag))
378                 {
379                     return false;
380                 }
381             }
382         }
383         throw new IOException("Unexpected end of stream");
384     }
385 
386     /**
387      * Read the attributes into an array and return the array. The position of each attribute is indicated by the vararg
388      * parameter 'attributes'.
389      * @param xmlr XMLStreamReader; the XML stream reader
390      * @param attributes String...; the attributes that are expected
391      * @return the array of atribute values, in the order of the vararg parameter 'attributes'
392      * @throws XMLStreamException on error reading from the XML stream
393      * @throws IOException when the current element does not contain the right (number of) attributes
394      */
395     private static String[] getAttributes(final XMLStreamReader xmlr, final String... attributes)
396             throws XMLStreamException, IOException
397     {
398         String[] result = new String[attributes.length];
399         int found = 0;
400         for (int i = 0; i < xmlr.getAttributeCount(); i++)
401         {
402             String localName = xmlr.getAttributeLocalName(i);
403             String value = xmlr.getAttributeValue(i);
404             for (int j = 0; j < attributes.length; j++)
405             {
406                 if (localName.equals(attributes[j]))
407                 {
408                     result[j] = value;
409                     found++;
410                 }
411             }
412         }
413         Throw.when(found != attributes.length, IOException.class, "attribute data does not contain %d fields",
414                 attributes.length);
415         return result;
416     }
417 
418     /**
419      * Read the data from the XML-file into the data table. Use the metadata to reconstruct the data table.
420      * @param filename String; the file name to read the data from
421      * @return dataTable the data table reconstructed from the meta data and filled with the data
422      * @throws IOException on I/O error when reading the data
423      * @throws TextSerializationException on unknown data type for serialization
424      * @throws XMLStreamException on XML read error
425      */
426     public static Table readData(final String filename) throws IOException, TextSerializationException, XMLStreamException
427     {
428         try (FileReader fr = new FileReader(filename))
429         {
430             return readData(fr);
431         }
432     }
433 
434 }