Routino SVN Repository Browser

Check out the latest version of Routino: svn co http://routino.org/svn/trunk routino

/[routino]/trunk/src/planetsplitter.c
ViewVC logotype

Contents of /trunk/src/planetsplitter.c

Parent Directory Parent Directory | Revision Log Revision Log

Revision 1221 - (show annotations) (download) (as text)
Fri Dec 21 16:08:44 2012 UTC (12 years, 2 months ago) by amb
File MIME type: text/x-csrc
File size: 20340 byte(s) 
Add a parser for OSM PBF format.
Separate the XML parser from the data processing in osmparser.c.
Update planetsplitter and documentation to use new format.
1 /***************************************
2 OSM planet file splitter.
3
4 Part of the Routino routing software.
5 ******************/ /******************
6 This file Copyright 2008-2012 Andrew M. Bishop
7
8 This program is free software: you can redistribute it and/or modify
9 it under the terms of the GNU Affero General Public License as published by
10 the Free Software Foundation, either version 3 of the License, or
11 (at your option) any later version.
12
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU Affero General Public License for more details.
17
18 You should have received a copy of the GNU Affero General Public License
19 along with this program. If not, see <http://www.gnu.org/licenses/>.
20 ***************************************/
21
22
23 #include <stdio.h>
24 #include <stdlib.h>
25 #include <unistd.h>
26 #include <string.h>
27 #include <errno.h>
28 #include <sys/time.h>
29
30 #include "types.h"
31 #include "ways.h"
32
33 #include "typesx.h"
34 #include "nodesx.h"
35 #include "segmentsx.h"
36 #include "waysx.h"
37 #include "relationsx.h"
38 #include "superx.h"
39 #include "prunex.h"
40
41 #include "files.h"
42 #include "logging.h"
43 #include "functions.h"
44 #include "osmparser.h"
45 #include "tagging.h"
46 #include "uncompress.h"
47
48
49 /* Global variables */
50
51 /*+ The name of the temporary directory. +*/
52 char *option_tmpdirname=NULL;
53
54 /*+ The amount of RAM to use for filesorting. +*/
55 size_t option_filesort_ramsize=0;
56
57 /*+ The number of threads to use for filesorting. +*/
58 int option_filesort_threads=1;
59
60
61 /* Local functions */
62
63 static void print_usage(int detail,const char *argerr,const char *err);
64
65
66 /*++++++++++++++++++++++++++++++++++++++
67 The main program for the planetsplitter.
68 ++++++++++++++++++++++++++++++++++++++*/
69
70 int main(int argc,char** argv)
71 {
72 struct timeval start_time;
73 NodesX *Nodes;
74 SegmentsX *Segments,*SuperSegments=NULL,*MergedSegments=NULL;
75 WaysX *Ways;
76 RelationsX *Relations;
77 int iteration=0,quit=0;
78 int max_iterations=5;
79 char *dirname=NULL,*prefix=NULL,*tagging=NULL,*errorlog=NULL;
80 int option_parse_only=0,option_process_only=0;
81 int option_append=0,option_keep=0,option_changes=0;
82 int option_filenames=0;
83 int option_prune_isolated=500,option_prune_short=5,option_prune_straight=3;
84 int arg;
85
86 gettimeofday(&start_time,NULL);
87
88 /* Parse the command line arguments */
89
90 for(arg=1;arg<argc;arg++)
91 {
92 if(!strcmp(argv[arg],"--help"))
93 print_usage(1,NULL,NULL);
94 else if(!strncmp(argv[arg],"--dir=",6))
95 dirname=&argv[arg][6];
96 else if(!strncmp(argv[arg],"--prefix=",9))
97 prefix=&argv[arg][9];
98 else if(!strncmp(argv[arg],"--sort-ram-size=",16))
99 option_filesort_ramsize=atoi(&argv[arg][16]);
100 #if defined(USE_PTHREADS) && USE_PTHREADS
101 else if(!strncmp(argv[arg],"--sort-threads=",15))
102 option_filesort_threads=atoi(&argv[arg][15]);
103 #endif
104 else if(!strncmp(argv[arg],"--tmpdir=",9))
105 option_tmpdirname=&argv[arg][9];
106 else if(!strncmp(argv[arg],"--tagging=",10))
107 tagging=&argv[arg][10];
108 else if(!strcmp(argv[arg],"--loggable"))
109 option_loggable=1;
110 else if(!strcmp(argv[arg],"--logtime"))
111 option_logtime=1;
112 else if(!strcmp(argv[arg],"--errorlog"))
113 errorlog="error.log";
114 else if(!strncmp(argv[arg],"--errorlog=",11))
115 errorlog=&argv[arg][11];
116 else if(!strcmp(argv[arg],"--parse-only"))
117 option_parse_only=1;
118 else if(!strcmp(argv[arg],"--process-only"))
119 option_process_only=1;
120 else if(!strcmp(argv[arg],"--append"))
121 option_append=1;
122 else if(!strcmp(argv[arg],"--keep"))
123 option_keep=1;
124 else if(!strcmp(argv[arg],"--changes"))
125 option_changes=1;
126 else if(!strncmp(argv[arg],"--max-iterations=",17))
127 max_iterations=atoi(&argv[arg][17]);
128 else if(!strncmp(argv[arg],"--prune",7))
129 {
130 if(!strcmp(&argv[arg][7],"-none"))
131 option_prune_isolated=option_prune_short=option_prune_straight=0;
132 else if(!strncmp(&argv[arg][7],"-isolated=",10))
133 option_prune_isolated=atoi(&argv[arg][17]);
134 else if(!strncmp(&argv[arg][7],"-short=",7))
135 option_prune_short=atoi(&argv[arg][14]);
136 else if(!strncmp(&argv[arg][7],"-straight=",10))
137 option_prune_straight=atoi(&argv[arg][17]);
138 else
139 print_usage(0,argv[arg],NULL);
140 }
141 else if(argv[arg][0]=='-' && argv[arg][1]=='-')
142 print_usage(0,argv[arg],NULL);
143 else
144 option_filenames++;
145 }
146
147 /* Check the specified command line options */
148
149 if(option_parse_only && option_process_only)
150 print_usage(0,NULL,"Cannot use '--parse-only' and '--process-only' at the same time.");
151
152 if(option_append && option_process_only)
153 print_usage(0,NULL,"Cannot use '--append' and '--process-only' at the same time.");
154
155 if(option_filenames && option_process_only)
156 print_usage(0,NULL,"Cannot use '--process-only' and filenames at the same time.");
157
158 if(!option_filesort_ramsize)
159 {
160 #if SLIM
161 option_filesort_ramsize=64*1024*1024;
162 #else
163 option_filesort_ramsize=256*1024*1024;
164 #endif
165 }
166 else
167 option_filesort_ramsize*=1024*1024;
168
169 if(!option_tmpdirname)
170 {
171 if(!dirname)
172 option_tmpdirname=".";
173 else
174 option_tmpdirname=dirname;
175 }
176
177 if(!option_process_only)
178 {
179 if(tagging)
180 {
181 if(!ExistsFile(tagging))
182 {
183 fprintf(stderr,"Error: The '--tagging' option specifies a file that does not exist.\n");
184 return(1);
185 }
186 }
187 else
188 {
189 if(ExistsFile(FileName(dirname,prefix,"tagging.xml")))
190 tagging=FileName(dirname,prefix,"tagging.xml");
191 else if(ExistsFile(FileName(DATADIR,NULL,"tagging.xml")))
192 tagging=FileName(DATADIR,NULL,"tagging.xml");
193 else
194 {
195 fprintf(stderr,"Error: The '--tagging' option was not used and the default 'tagging.xml' does not exist.\n");
196 return(1);
197 }
198 }
199
200 if(ParseXMLTaggingRules(tagging))
201 {
202 fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging);
203 return(1);
204 }
205 }
206
207 /* Create new node, segment, way and relation variables */
208
209 Nodes=NewNodeList(option_append||option_changes,option_process_only);
210
211 Segments=NewSegmentList(option_append||option_changes,option_process_only);
212
213 Ways=NewWayList(option_append||option_changes,option_process_only);
214
215 Relations=NewRelationList(option_append||option_changes,option_process_only);
216
217 /* Create the error log file */
218
219 if(errorlog)
220 open_errorlog(FileName(dirname,prefix,errorlog),option_append||option_changes||option_process_only);
221
222 /* Parse the file */
223
224 if(!option_process_only)
225 {
226 if(option_filenames)
227 {
228 for(arg=1;arg<argc;arg++)
229 {
230 int fd;
231 char *p;
232
233 if(argv[arg][0]=='-' && argv[arg][1]=='-')
234 continue;
235
236 fd=ReOpenFile(argv[arg]);
237
238 if((p=strstr(argv[arg],".bz2")) && !strcmp(p,".bz2"))
239 fd=Uncompress_Bzip2(fd);
240
241 if((p=strstr(argv[arg],".gz")) && !strcmp(p,".gz"))
242 fd=Uncompress_Gzip(fd);
243
244 if(option_changes)
245 {
246 printf("\nParse OSC Data [%s]\n==============\n\n",argv[arg]);
247 fflush(stdout);
248
249 if((p=strstr(argv[arg],".pbf")) && !strcmp(p,".pbf"))
250 {
251 if(ParsePBFFile(fd,Nodes,Segments,Ways,Relations,option_changes))
252 exit(EXIT_FAILURE);
253 }
254 else
255 {
256 if(ParseOSCFile(fd,Nodes,Segments,Ways,Relations))
257 exit(EXIT_FAILURE);
258 }
259 }
260 else
261 {
262 printf("\nParse OSM Data [%s]\n==============\n\n",argv[arg]);
263 fflush(stdout);
264
265 if((p=strstr(argv[arg],".pbf")) && !strcmp(p,".pbf"))
266 {
267 if(ParsePBFFile(fd,Nodes,Segments,Ways,Relations,option_changes))
268 exit(EXIT_FAILURE);
269 }
270 else
271 {
272 if(ParseOSMFile(fd,Nodes,Segments,Ways,Relations))
273 exit(EXIT_FAILURE);
274 }
275 }
276
277 CloseFile(fd);
278 }
279 }
280 else
281 {
282 if(option_changes)
283 {
284 printf("\nParse OSC Data\n==============\n\n");
285 fflush(stdout);
286
287 if(ParseOSCFile(STDIN_FILENO,Nodes,Segments,Ways,Relations))
288 exit(EXIT_FAILURE);
289 }
290 else
291 {
292 printf("\nParse OSM Data\n==============\n\n");
293 fflush(stdout);
294
295 if(ParseOSMFile(STDIN_FILENO,Nodes,Segments,Ways,Relations))
296 exit(EXIT_FAILURE);
297 }
298 }
299
300 DeleteXMLTaggingRules();
301 }
302
303 FinishNodeList(Nodes);
304 FinishSegmentList(Segments);
305 FinishWayList(Ways);
306 FinishRelationList(Relations);
307
308 if(option_parse_only)
309 {
310 FreeNodeList(Nodes,1);
311 FreeSegmentList(Segments,1);
312 FreeWayList(Ways,1);
313 FreeRelationList(Relations,1);
314
315 return(0);
316 }
317
318
319 /* Sort the data */
320
321 printf("\nSort OSM Data\n=============\n\n");
322 fflush(stdout);
323
324 /* Sort the nodes, segments, ways and relations */
325
326 SortNodeList(Nodes);
327
328 if(option_changes)
329 ApplySegmentChanges(Segments);
330
331 SortSegmentList(Segments);
332
333 SortWayList(Ways);
334
335 SortRelationList(Relations);
336
337 /* Process the data */
338
339 printf("\nProcess OSM Data\n================\n\n");
340 fflush(stdout);
341
342 /* Extract the way names (must be before using the ways) */
343
344 ExtractWayNames(Ways,option_keep||option_changes);
345
346 /* Remove bad segments (must be after sorting the nodes, segments and ways) */
347
348 RemoveBadSegments(Segments,Nodes,Ways,option_keep||option_changes);
349
350 /* Remove non-highway nodes (must be after removing the bad segments) */
351
352 RemoveNonHighwayNodes(Nodes,Segments,option_keep||option_changes);
353
354 /* Process the route relations and first part of turn relations (must be before compacting the ways) */
355
356 ProcessRouteRelations(Relations,Ways,option_keep||option_changes);
357
358 ProcessTurnRelations1(Relations,Nodes,Ways,option_keep||option_changes);
359
360 /* Measure the segments and replace node/way id with index (must be after removing non-highway nodes) */
361
362 MeasureSegments(Segments,Nodes,Ways);
363
364 /* Index the segments */
365
366 IndexSegments(Segments,Nodes,Ways);
367
368 /* Convert the turn relations from ways into nodes */
369
370 ProcessTurnRelations2(Relations,Nodes,Segments,Ways);
371
372 /* Compact the ways (must be after turn relations 2) */
373
374 CompactWayList(Ways,Segments);
375
376 /* Index the segments */
377
378 IndexSegments(Segments,Nodes,Ways);
379
380 /* Prune unwanted nodes/segments. */
381
382 if(option_prune_straight || option_prune_isolated || option_prune_short)
383 {
384 printf("\nPrune Unneeded Data\n===================\n\n");
385 fflush(stdout);
386
387 StartPruning(Nodes,Segments,Ways);
388
389 if(option_prune_straight)
390 PruneStraightHighwayNodes(Nodes,Segments,Ways,option_prune_straight);
391
392 if(option_prune_isolated)
393 PruneIsolatedRegions(Nodes,Segments,Ways,option_prune_isolated);
394
395 if(option_prune_short)
396 PruneShortSegments(Nodes,Segments,Ways,option_prune_short);
397
398 FinishPruning(Nodes,Segments,Ways);
399
400 /* Remove the pruned nodes and segments and update the indexes */
401
402 RemovePrunedNodes(Nodes,Segments);
403 RemovePrunedSegments(Segments,Ways);
404 CompactWayList(Ways,Segments);
405 RemovePrunedTurnRelations(Relations,Nodes);
406 IndexSegments(Segments,Nodes,Ways);
407 }
408
409 /* Repeated iteration on Super-Nodes and Super-Segments */
410
411 do
412 {
413 int nsuper;
414
415 printf("\nProcess Super-Data (iteration %d)\n================================%s\n\n",iteration,iteration>9?"=":"");
416 fflush(stdout);
417
418 if(iteration==0)
419 {
420 /* Select the super-nodes */
421
422 ChooseSuperNodes(Nodes,Segments,Ways);
423
424 /* Select the super-segments */
425
426 SuperSegments=CreateSuperSegments(Nodes,Segments,Ways);
427
428 nsuper=Segments->number;
429 }
430 else
431 {
432 SegmentsX *SuperSegments2;
433
434 /* Select the super-nodes */
435
436 ChooseSuperNodes(Nodes,SuperSegments,Ways);
437
438 /* Select the super-segments */
439
440 SuperSegments2=CreateSuperSegments(Nodes,SuperSegments,Ways);
441
442 nsuper=SuperSegments->number;
443
444 FreeSegmentList(SuperSegments,0);
445
446 SuperSegments=SuperSegments2;
447 }
448
449 /* Sort the super-segments and remove duplicates */
450
451 DeduplicateSuperSegments(SuperSegments,Ways);
452
453 /* Index the segments */
454
455 IndexSegments(SuperSegments,Nodes,Ways);
456
457 /* Check for end condition */
458
459 if(SuperSegments->number==nsuper)
460 quit=1;
461
462 iteration++;
463
464 if(iteration>max_iterations)
465 quit=1;
466 }
467 while(!quit);
468
469 /* Combine the super-segments */
470
471 printf("\nCombine Segments and Super-Segments\n===================================\n\n");
472 fflush(stdout);
473
474 /* Merge the super-segments */
475
476 MergedSegments=MergeSuperSegments(Segments,SuperSegments);
477
478 FreeSegmentList(Segments,0);
479
480 FreeSegmentList(SuperSegments,0);
481
482 Segments=MergedSegments;
483
484 /* Re-index the merged segments */
485
486 IndexSegments(Segments,Nodes,Ways);
487
488 /* Cross reference the nodes and segments */
489
490 printf("\nCross-Reference Nodes and Segments\n==================================\n\n");
491 fflush(stdout);
492
493 /* Sort the nodes and segments geographically */
494
495 SortNodeListGeographically(Nodes);
496
497 SortSegmentListGeographically(Segments,Nodes);
498
499 /* Re-index the segments */
500
501 IndexSegments(Segments,Nodes,Ways);
502
503 /* Sort the turn relations geographically */
504
505 SortTurnRelationListGeographically(Relations,Nodes,Segments);
506
507 /* Output the results */
508
509 printf("\nWrite Out Database Files\n========================\n\n");
510 fflush(stdout);
511
512 /* Write out the nodes */
513
514 SaveNodeList(Nodes,FileName(dirname,prefix,"nodes.mem"),Segments);
515
516 FreeNodeList(Nodes,0);
517
518 /* Write out the segments */
519
520 SaveSegmentList(Segments,FileName(dirname,prefix,"segments.mem"));
521
522 FreeSegmentList(Segments,0);
523
524 /* Write out the ways */
525
526 SaveWayList(Ways,FileName(dirname,prefix,"ways.mem"));
527
528 FreeWayList(Ways,0);
529
530 /* Write out the relations */
531
532 SaveRelationList(Relations,FileName(dirname,prefix,"relations.mem"));
533
534 FreeRelationList(Relations,0);
535
536 /* Close the error log file */
537
538 if(errorlog)
539 close_errorlog();
540
541 /* Print the total time */
542
543 if(option_logtime)
544 {
545 printf("\n");
546 fprintf_elapsed_time(stdout,&start_time);
547 printf("Complete\n");
548 fflush(stdout);
549 }
550
551 return(0);
552 }
553
554
555 /*++++++++++++++++++++++++++++++++++++++
556 Print out the usage information.
557
558 int detail The level of detail to use - 0 = low, 1 = high.
559
560 const char *argerr The argument that gave the error (if there is one).
561
562 const char *err Other error message (if there is one).
563 ++++++++++++++++++++++++++++++++++++++*/
564
565 static void print_usage(int detail,const char *argerr,const char *err)
566 {
567 fprintf(stderr,
568 "Usage: planetsplitter [--help]\n"
569 " [--dir=<dirname>] [--prefix=<name>]\n"
570 #if defined(USE_PTHREADS) && USE_PTHREADS
571 " [--sort-ram-size=<size>] [--sort-threads=<number>]\n"
572 #else
573 " [--sort-ram-size=<size>]\n"
574 #endif
575 " [--tmpdir=<dirname>]\n"
576 " [--tagging=<filename>]\n"
577 " [--loggable] [--logtime]\n"
578 " [--errorlog[=<name>]]\n"
579 " [--parse-only | --process-only]\n"
580 " [--append] [--keep] [--changes]\n"
581 " [--max-iterations=<number>]\n"
582 " [--prune-none]\n"
583 " [--prune-isolated=<len>]\n"
584 " [--prune-short=<len>]\n"
585 " [--prune-straight=<len>]\n"
586 " [<filename.osm> ... | <filename.osc> ...\n"
587 " | <filename.osm.pbf> ..."
588 #if defined(USE_BZIP2) && USE_BZIP2
589 "\n | <filename.osm.bz2> ... | <filename.osc.bz2> ..."
590 #endif
591 #if defined(USE_GZIP) && USE_GZIP
592 "\n | <filename.osm.gz> ... | <filename.osc.gz> ..."
593 #endif
594 "]\n");
595
596 if(argerr)
597 fprintf(stderr,
598 "\n"
599 "Error with command line parameter: %s\n",argerr);
600
601 if(err)
602 fprintf(stderr,
603 "\n"
604 "Error: %s\n",err);
605
606 if(detail)
607 fprintf(stderr,
608 "\n"
609 "--help Prints this information.\n"
610 "\n"
611 "--dir=<dirname> The directory containing the routing database.\n"
612 "--prefix=<name> The filename prefix for the routing database.\n"
613 "\n"
614 "--sort-ram-size=<size> The amount of RAM (in MB) to use for data sorting\n"
615 #if SLIM
616 " (defaults to 64MB otherwise.)\n"
617 #else
618 " (defaults to 256MB otherwise.)\n"
619 #endif
620 #if defined(USE_PTHREADS) && USE_PTHREADS
621 "--sort-threads=<number> The number of threads to use for data sorting.\n"
622 #endif
623 "\n"
624 "--tmpdir=<dirname> The directory name for temporary files.\n"
625 " (defaults to the '--dir' option directory.)\n"
626 "\n"
627 "--tagging=<filename> The name of the XML file containing the tagging rules\n"
628 " (defaults to 'tagging.xml' with '--dir' and\n"
629 " '--prefix' options or the file installed in\n"
630 " '" DATADIR "').\n"
631 "\n"
632 "--loggable Print progress messages suitable for logging to file.\n"
633 "--logtime Print the elapsed time for each processing step.\n"
634 "--errorlog[=<name>] Log parsing errors to 'error.log' or the given name\n"
635 " (the '--dir' and '--prefix' options are applied).\n"
636 "\n"
637 "--parse-only Parse the OSM/OSC file(s) and store the results.\n"
638 "--process-only Process the stored results from previous option.\n"
639 "--append Parse the OSM file(s) and append to existing results.\n"
640 "--keep Keep the intermediate files after parsing & sorting.\n"
641 "--changes Parse the data as an OSC file and apply the changes.\n"
642 "\n"
643 "--max-iterations=<number> The number of iterations for finding super-nodes\n"
644 " (defaults to 5).\n"
645 "\n"
646 "--prune-none Disable the prune options below, they are re-enabled\n"
647 " by adding them to the command line after this option.\n"
648 "--prune-isolated=<len> Remove access from small disconnected segment groups\n"
649 " (defaults to removing groups under 500m).\n"
650 "--prune-short=<len> Remove short segments (defaults to removing segments\n"
651 " up to a maximum length of 5m).\n"
652 "--prune-straight=<len> Remove nodes in almost straight highways (defaults to\n"
653 " removing nodes up to 3m offset from a straight line).\n"
654 "\n"
655 "<filename.osm> ... The name(s) of the file(s) to process (defaults to\n"
656 "<filename.osc> ... reading data from standard input).\n"
657 "<filename.osm.pbf> ... Filenames ending '.pbf' read as PBF, others as XML.\n"
658 #if defined(USE_BZIP2) && USE_BZIP2
659 " Filenames ending '.bz2' will be bzip2 uncompressed.\n"
660 #endif
661 #if defined(USE_GZIP) && USE_GZIP
662 " Filenames ending '.gz' will be gzip uncompressed.\n"
663 #endif
664 "\n"
665 "<transport> defaults to all but can be set to:\n"
666 "%s"
667 "\n"
668 "<highway> can be selected from:\n"
669 "%s"
670 "\n"
671 "<property> can be selected from:\n"
672 "%s",
673 TransportList(),HighwayList(),PropertyList());
674
675 exit(!detail);
676 }

Properties

Name Value
cvs:description Planet file splitter.
Index: This Site Routino Software Download Source Code Documentation XML Files Links
Index: View SVN Development Recent changes Development (destination-access branch) Version 3.4.1 (destination-access branch) Version 3.4.1 Version 3.4 (destination-access branch) Version 3.4 Version 3.3.3 (destination-access branch) Version 3.3.3 Version 3.3.2 (destination-access branch) Version 3.3.2 Version 3.3.1 (destination-access branch) Version 3.3.1 Version 3.3 (destination-access branch) Version 3.3 Version 3.2 Version 3.1.1 Version 3.1 Version 3.0 Version 2.7.3 Version 2.7.2 Version 2.7.1 Version 2.7 Version 2.6 Version 2.5.1 Version 2.5 Version 2.4.1 Version 2.4 Version 2.3.2 Version 2.3.1 Version 2.3 Version 2.2 Version 2.1.2 Version 2.1.1 Version 2.1 Version 2.0.3 Version 2.0.2 Version 2.0.1 Version 2.0 Version 1.5.1 Version 1.5 Version 1.4.1 Version 1.4 Version 1.3 Version 1.2 Version 1.1 Version 1.0
Powered by ViewVC 1.3.0-dev ViewVC Help
© Andrew M. Bishop = < amb @ ... >
Root