QGIS API Documentation  2.8.2-Wien
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
qgsosmimport.cpp
Go to the documentation of this file.
1 /***************************************************************************
2  qgsosmimport.cpp
3  --------------------------------------
4  Date : January 2013
5  Copyright : (C) 2013 by Martin Dobias
6  Email : wonder dot sk at gmail dot com
7  ***************************************************************************
8  * *
9  * This program is free software; you can redistribute it and/or modify *
10  * it under the terms of the GNU General Public License as published by *
11  * the Free Software Foundation; either version 2 of the License, or *
12  * (at your option) any later version. *
13  * *
14  ***************************************************************************/
15 
16 #include "qgsosmimport.h"
17 
18 #include <spatialite.h>
19 
20 #include <QStringList>
21 #include <QXmlStreamReader>
22 
23 
24 QgsOSMXmlImport::QgsOSMXmlImport( const QString& xmlFilename, const QString& dbFilename )
25  : mXmlFileName( xmlFilename )
26  , mDbFileName( dbFilename )
27  , mDatabase( 0 )
28  , mStmtInsertNode( 0 )
29  , mStmtInsertNodeTag( 0 )
30  , mStmtInsertWay( 0 )
31  , mStmtInsertWayNode( 0 )
32  , mStmtInsertWayTag( 0 )
33 {
34 
35 }
36 
37 bool QgsOSMXmlImport::import()
38 {
39  mError.clear();
40 
41  // open input
42  mInputFile.setFileName( mXmlFileName );
43  if ( !mInputFile.open( QIODevice::ReadOnly ) )
44  {
45  mError = QString( "Cannot open input file: %1" ).arg( mXmlFileName );
46  return false;
47  }
48 
49  // open output
50 
51  if ( QFile::exists( mDbFileName ) )
52  {
53  if ( !QFile( mDbFileName ).remove() )
54  {
55  mError = QString( "Database file cannot be overwritten: %1" ).arg( mDbFileName );
56  return false;
57  }
58  }
59 
60  // load spatialite extension
61  spatialite_init( 0 );
62 
63  if ( !createDatabase() )
64  {
65  // mError is set in createDatabase()
66  return false;
67  }
68 
69  qDebug( "starting import" );
70 
71  int retX = sqlite3_exec( mDatabase, "BEGIN", NULL, NULL, 0 );
72  Q_ASSERT( retX == SQLITE_OK );
73  Q_UNUSED( retX );
74 
75  // start parsing
76 
77  QXmlStreamReader xml( &mInputFile );
78 
79  while ( !xml.atEnd() )
80  {
81  xml.readNext();
82 
83  if ( xml.isEndDocument() )
84  break;
85 
86  if ( xml.isStartElement() )
87  {
88  if ( xml.name() == "osm" )
89  readRoot( xml );
90  else
91  xml.raiseError( "Invalid root tag" );
92  }
93  }
94 
95  int retY = sqlite3_exec( mDatabase, "COMMIT", NULL, NULL, 0 );
96  Q_ASSERT( retY == SQLITE_OK );
97  Q_UNUSED( retY );
98 
99  createIndexes();
100 
101  if ( xml.hasError() )
102  {
103  mError = QString( "XML error: %1" ).arg( xml.errorString() );
104  return false;
105  }
106 
107  closeDatabase();
108 
109  return true;
110 }
111 
112 bool QgsOSMXmlImport::createIndexes()
113 {
114  // index on tags for faster access
115  const char* sqlIndexes[] =
116  {
117  "CREATE INDEX nodes_tags_idx ON nodes_tags(id)",
118  "CREATE INDEX ways_tags_idx ON ways_tags(id)",
119  "CREATE INDEX ways_nodes_way ON ways_nodes(way_id)"
120  };
121  int count = sizeof( sqlIndexes ) / sizeof( const char* );
122  for ( int i = 0; i < count; ++i )
123  {
124  int ret = sqlite3_exec( mDatabase, sqlIndexes[i], 0, 0, 0 );
125  if ( ret != SQLITE_OK )
126  {
127  mError = "Error creating indexes!";
128  return false;
129  }
130  }
131 
132  return true;
133 }
134 
135 
136 bool QgsOSMXmlImport::createDatabase()
137 {
138  char **results;
139  int rows, columns;
140  if ( sqlite3_open_v2( mDbFileName.toUtf8().data(), &mDatabase, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0 ) != SQLITE_OK )
141  return false;
142 
143  bool above41 = false;
144  int ret = sqlite3_get_table( mDatabase, "select spatialite_version()", &results, &rows, &columns, NULL );
145  if ( ret == SQLITE_OK && rows == 1 && columns == 1 )
146  {
147  QString version = QString::fromUtf8( results[1] );
148  QStringList parts = version.split( " ", QString::SkipEmptyParts );
149  if ( parts.size() >= 1 )
150  {
151  QStringList verparts = parts[0].split( ".", QString::SkipEmptyParts );
152  above41 = verparts.size() >= 2 && ( verparts[0].toInt() > 4 || ( verparts[0].toInt() == 4 && verparts[1].toInt() >= 1 ) );
153  }
154  }
155  sqlite3_free_table( results );
156 
157  const char* sqlInitStatements[] =
158  {
159  "PRAGMA cache_size = 100000", // TODO!!!
160  "PRAGMA synchronous = OFF", // TODO!!!
161  above41 ? "SELECT InitSpatialMetadata(1)" : "SELECT InitSpatialMetadata()",
162  "CREATE TABLE nodes ( id INTEGER PRIMARY KEY, lat REAL, lon REAL )",
163  "CREATE TABLE nodes_tags ( id INTEGER, k TEXT, v TEXT )",
164  "CREATE TABLE ways ( id INTEGER PRIMARY KEY )",
165  "CREATE TABLE ways_nodes ( way_id INTEGER, node_id INTEGER, way_pos INTEGER )",
166  "CREATE TABLE ways_tags ( id INTEGER, k TEXT, v TEXT )",
167  };
168 
169  int initCount = sizeof( sqlInitStatements ) / sizeof( const char* );
170  for ( int i = 0; i < initCount; ++i )
171  {
172  char* errMsg;
173  if ( sqlite3_exec( mDatabase, sqlInitStatements[i], 0, 0, &errMsg ) != SQLITE_OK )
174  {
175  mError = QString( "Error executing SQL command:\n%1\nSQL:\n%2" )
176  .arg( QString::fromUtf8( errMsg ) ).arg( QString::fromUtf8( sqlInitStatements[i] ) );
177  sqlite3_free( errMsg );
178  closeDatabase();
179  return false;
180  }
181  }
182 
183  const char* sqlInsertStatements[] =
184  {
185  "INSERT INTO nodes ( id, lat, lon ) VALUES (?,?,?)",
186  "INSERT INTO nodes_tags ( id, k, v ) VALUES (?,?,?)",
187  "INSERT INTO ways ( id ) VALUES (?)",
188  "INSERT INTO ways_nodes ( way_id, node_id, way_pos ) VALUES (?,?,?)",
189  "INSERT INTO ways_tags ( id, k, v ) VALUES (?,?,?)"
190  };
191  sqlite3_stmt** sqliteInsertStatements[] =
192  {
193  &mStmtInsertNode,
194  &mStmtInsertNodeTag,
195  &mStmtInsertWay,
196  &mStmtInsertWayNode,
197  &mStmtInsertWayTag
198  };
199  Q_ASSERT( sizeof( sqlInsertStatements ) / sizeof( const char* ) == sizeof( sqliteInsertStatements ) / sizeof( sqlite3_stmt** ) );
200  int insertCount = sizeof( sqlInsertStatements ) / sizeof( const char* );
201 
202  for ( int i = 0; i < insertCount; ++i )
203  {
204  if ( sqlite3_prepare_v2( mDatabase, sqlInsertStatements[i], -1, sqliteInsertStatements[i], 0 ) != SQLITE_OK )
205  {
206  const char* errMsg = sqlite3_errmsg( mDatabase ); // does not require free
207  mError = QString( "Error preparing SQL command:\n%1\nSQL:\n%2" )
208  .arg( QString::fromUtf8( errMsg ) ).arg( QString::fromUtf8( sqlInsertStatements[i] ) );
209  closeDatabase();
210  return false;
211  }
212  }
213 
214  return true;
215 }
216 
217 
218 void QgsOSMXmlImport::deleteStatement( sqlite3_stmt*& stmt )
219 {
220  if ( stmt )
221  {
222  sqlite3_finalize( stmt );
223  stmt = 0;
224  }
225 }
226 
227 
228 bool QgsOSMXmlImport::closeDatabase()
229 {
230  if ( !mDatabase )
231  return false;
232 
233  deleteStatement( mStmtInsertNode );
234  deleteStatement( mStmtInsertNodeTag );
235  deleteStatement( mStmtInsertWay );
236  deleteStatement( mStmtInsertWayNode );
237  deleteStatement( mStmtInsertWayTag );
238 
239  Q_ASSERT( mStmtInsertNode == 0 );
240 
241  sqlite3_close( mDatabase );
242  mDatabase = 0;
243  return true;
244 }
245 
246 
247 void QgsOSMXmlImport::readRoot( QXmlStreamReader& xml )
248 {
249  int i = 0;
250  int percent = -1;
251 
252  while ( !xml.atEnd() )
253  {
254  xml.readNext();
255 
256  if ( xml.isEndElement() ) // </osm>
257  break;
258 
259  if ( xml.isStartElement() )
260  {
261  if ( ++i == 500 )
262  {
263  int new_percent = 100 * mInputFile.pos() / mInputFile.size();
264  if ( new_percent > percent )
265  {
266  emit progress( new_percent );
267  percent = new_percent;
268  }
269  i = 0;
270  }
271 
272  if ( xml.name() == "node" )
273  readNode( xml );
274  else if ( xml.name() == "way" )
275  readWay( xml );
276  else
277  xml.skipCurrentElement();
278  }
279  }
280 }
281 
282 
283 void QgsOSMXmlImport::readNode( QXmlStreamReader& xml )
284 {
285  // <node id="2197214" lat="50.0682113" lon="14.4348483" user="viduka" uid="595326" visible="true" version="10" changeset="10714591" timestamp="2012-02-17T19:58:49Z">
286  QXmlStreamAttributes attrs = xml.attributes();
287  QgsOSMId id = attrs.value( "id" ).toString().toLongLong();
288  double lat = attrs.value( "lat" ).toString().toDouble();
289  double lon = attrs.value( "lon" ).toString().toDouble();
290 
291  // insert to DB
292  sqlite3_bind_int64( mStmtInsertNode, 1, id );
293  sqlite3_bind_double( mStmtInsertNode, 2, lat );
294  sqlite3_bind_double( mStmtInsertNode, 3, lon );
295 
296  if ( sqlite3_step( mStmtInsertNode ) != SQLITE_DONE )
297  {
298  xml.raiseError( QString( "Storing node %1 failed." ).arg( id ) );
299  }
300 
301  sqlite3_reset( mStmtInsertNode );
302 
303  while ( !xml.atEnd() )
304  {
305  xml.readNext();
306 
307  if ( xml.isEndElement() ) // </node>
308  break;
309 
310  if ( xml.isStartElement() )
311  {
312  if ( xml.name() == "tag" )
313  readTag( false, id, xml );
314  else
315  xml.raiseError( "Invalid tag in <node>" );
316  }
317  }
318 }
319 
320 void QgsOSMXmlImport::readTag( bool way, QgsOSMId id, QXmlStreamReader& xml )
321 {
322  QXmlStreamAttributes attrs = xml.attributes();
323  QByteArray k = attrs.value( "k" ).toString().toUtf8();
324  QByteArray v = attrs.value( "v" ).toString().toUtf8();
325  xml.skipCurrentElement();
326 
327  sqlite3_stmt* stmtInsertTag = way ? mStmtInsertWayTag : mStmtInsertNodeTag;
328 
329  sqlite3_bind_int64( stmtInsertTag, 1, id );
330  sqlite3_bind_text( stmtInsertTag, 2, k.constData(), -1, SQLITE_STATIC );
331  sqlite3_bind_text( stmtInsertTag, 3, v.constData(), -1, SQLITE_STATIC );
332 
333  int res = sqlite3_step( stmtInsertTag );
334  if ( res != SQLITE_DONE )
335  {
336  xml.raiseError( QString( "Storing tag failed [%1]" ).arg( res ) );
337  }
338 
339  sqlite3_reset( stmtInsertTag );
340 }
341 
342 void QgsOSMXmlImport::readWay( QXmlStreamReader& xml )
343 {
344  /*
345  <way id="141756602" user="Vratislav Filler" uid="527259" visible="true" version="1" changeset="10145142" timestamp="2011-12-18T10:43:14Z">
346  <nd ref="318529958"/>
347  <nd ref="1551725779"/>
348  <nd ref="1551725792"/>
349  <nd ref="809695938"/>
350  <nd ref="1551725689"/>
351  <nd ref="809695935"/>
352  <tag k="highway" v="service"/>
353  <tag k="oneway" v="yes"/>
354  </way>
355  */
356  QXmlStreamAttributes attrs = xml.attributes();
357  QgsOSMId id = attrs.value( "id" ).toString().toLongLong();
358 
359  // insert to DB
360  sqlite3_bind_int64( mStmtInsertWay, 1, id );
361 
362  if ( sqlite3_step( mStmtInsertWay ) != SQLITE_DONE )
363  {
364  xml.raiseError( QString( "Storing way %1 failed." ).arg( id ) );
365  }
366 
367  sqlite3_reset( mStmtInsertWay );
368 
369  int way_pos = 0;
370 
371  while ( !xml.atEnd() )
372  {
373  xml.readNext();
374 
375  if ( xml.isEndElement() ) // </way>
376  break;
377 
378  if ( xml.isStartElement() )
379  {
380  if ( xml.name() == "nd" )
381  {
382  QgsOSMId node_id = xml.attributes().value( "ref" ).toString().toLongLong();
383 
384  sqlite3_bind_int64( mStmtInsertWayNode, 1, id );
385  sqlite3_bind_int64( mStmtInsertWayNode, 2, node_id );
386  sqlite3_bind_int( mStmtInsertWayNode, 3, way_pos );
387 
388  if ( sqlite3_step( mStmtInsertWayNode ) != SQLITE_DONE )
389  {
390  xml.raiseError( QString( "Storing ways_nodes %1 - %2 failed." ).arg( id ).arg( node_id ) );
391  }
392 
393  sqlite3_reset( mStmtInsertWayNode );
394 
395  way_pos++;
396 
397  xml.skipCurrentElement();
398  }
399  else if ( xml.name() == "tag" )
400  readTag( true, id, xml );
401  else
402  xml.skipCurrentElement();
403  }
404  }
405 }
Generated on Sun May 10 2015 16:33:23 for QGIS API Documentation by   doxygen 1.8.1.2