Routino SVN Repository Browser

Check out the latest version of Routino: svn co http://routino.org/svn/trunk routino

ViewVC logotype

Contents of /trunk/src/planetsplitter.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1348 - (show annotations) (download) (as text)
Mon May 27 18:29:44 2013 UTC (11 years, 9 months ago) by amb
File MIME type: text/x-csrc
File size: 21037 byte(s)
Simplify the sorting of the way names (only sort the names, not all the ways
twice), merge the generating of segments and separation of way names.

1 /***************************************
2 OSM planet file splitter.
3
4 Part of the Routino routing software.
5 ******************/ /******************
6 This file Copyright 2008-2013 Andrew M. Bishop
7
8 This program is free software: you can redistribute it and/or modify
9 it under the terms of the GNU Affero General Public License as published by
10 the Free Software Foundation, either version 3 of the License, or
11 (at your option) any later version.
12
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU Affero General Public License for more details.
17
18 You should have received a copy of the GNU Affero General Public License
19 along with this program. If not, see <http://www.gnu.org/licenses/>.
20 ***************************************/
21
22
23 #include <stdio.h>
24 #include <stdlib.h>
25 #include <unistd.h>
26 #include <string.h>
27 #include <errno.h>
28 #include <sys/time.h>
29
30 #include "types.h"
31 #include "ways.h"
32
33 #include "typesx.h"
34 #include "nodesx.h"
35 #include "segmentsx.h"
36 #include "waysx.h"
37 #include "relationsx.h"
38 #include "superx.h"
39 #include "prunex.h"
40
41 #include "files.h"
42 #include "logging.h"
43 #include "errorlogx.h"
44 #include "functions.h"
45 #include "osmparser.h"
46 #include "tagging.h"
47 #include "uncompress.h"
48
49
50 /* Global variables */
51
52 /*+ The name of the temporary directory. +*/
53 char *option_tmpdirname=NULL;
54
55 /*+ The amount of RAM to use for filesorting. +*/
56 size_t option_filesort_ramsize=0;
57
58 /*+ The number of threads to use for filesorting. +*/
59 int option_filesort_threads=1;
60
61
62 /* Local functions */
63
64 static void print_usage(int detail,const char *argerr,const char *err);
65
66
67 /*++++++++++++++++++++++++++++++++++++++
68 The main program for the planetsplitter.
69 ++++++++++++++++++++++++++++++++++++++*/
70
71 int main(int argc,char** argv)
72 {
73 struct timeval start_time;
74 NodesX *OSMNodes;
75 SegmentsX *OSMSegments,*SuperSegments=NULL,*MergedSegments=NULL;
76 WaysX *OSMWays;
77 RelationsX *OSMRelations;
78 int iteration=0,quit=0;
79 int max_iterations=5;
80 char *dirname=NULL,*prefix=NULL,*tagging=NULL,*errorlog=NULL;
81 int option_parse_only=0,option_process_only=0;
82 int option_append=0,option_keep=0,option_changes=0;
83 int option_filenames=0;
84 int option_prune_isolated=500,option_prune_short=5,option_prune_straight=3;
85 int arg;
86
87 gettimeofday(&start_time,NULL);
88
89 /* Parse the command line arguments */
90
91 for(arg=1;arg<argc;arg++)
92 {
93 if(!strcmp(argv[arg],"--help"))
94 print_usage(1,NULL,NULL);
95 else if(!strncmp(argv[arg],"--dir=",6))
96 dirname=&argv[arg][6];
97 else if(!strncmp(argv[arg],"--prefix=",9))
98 prefix=&argv[arg][9];
99 else if(!strncmp(argv[arg],"--sort-ram-size=",16))
100 option_filesort_ramsize=atoi(&argv[arg][16]);
101 #if defined(USE_PTHREADS) && USE_PTHREADS
102 else if(!strncmp(argv[arg],"--sort-threads=",15))
103 option_filesort_threads=atoi(&argv[arg][15]);
104 #endif
105 else if(!strncmp(argv[arg],"--tmpdir=",9))
106 option_tmpdirname=&argv[arg][9];
107 else if(!strncmp(argv[arg],"--tagging=",10))
108 tagging=&argv[arg][10];
109 else if(!strcmp(argv[arg],"--loggable"))
110 option_loggable=1;
111 else if(!strcmp(argv[arg],"--logtime"))
112 option_logtime=1;
113 else if(!strcmp(argv[arg],"--errorlog"))
114 errorlog="error.log";
115 else if(!strncmp(argv[arg],"--errorlog=",11))
116 errorlog=&argv[arg][11];
117 else if(!strcmp(argv[arg],"--parse-only"))
118 option_parse_only=1;
119 else if(!strcmp(argv[arg],"--process-only"))
120 option_process_only=1;
121 else if(!strcmp(argv[arg],"--append"))
122 option_append=1;
123 else if(!strcmp(argv[arg],"--keep"))
124 option_keep=1;
125 else if(!strcmp(argv[arg],"--changes"))
126 option_changes=1;
127 else if(!strncmp(argv[arg],"--max-iterations=",17))
128 max_iterations=atoi(&argv[arg][17]);
129 else if(!strncmp(argv[arg],"--prune",7))
130 {
131 if(!strcmp(&argv[arg][7],"-none"))
132 option_prune_isolated=option_prune_short=option_prune_straight=0;
133 else if(!strncmp(&argv[arg][7],"-isolated=",10))
134 option_prune_isolated=atoi(&argv[arg][17]);
135 else if(!strncmp(&argv[arg][7],"-short=",7))
136 option_prune_short=atoi(&argv[arg][14]);
137 else if(!strncmp(&argv[arg][7],"-straight=",10))
138 option_prune_straight=atoi(&argv[arg][17]);
139 else
140 print_usage(0,argv[arg],NULL);
141 }
142 else if(argv[arg][0]=='-' && argv[arg][1]=='-')
143 print_usage(0,argv[arg],NULL);
144 else
145 option_filenames++;
146 }
147
148 /* Check the specified command line options */
149
150 if(option_parse_only && option_process_only)
151 print_usage(0,NULL,"Cannot use '--parse-only' and '--process-only' at the same time.");
152
153 if(option_append && option_process_only)
154 print_usage(0,NULL,"Cannot use '--append' and '--process-only' at the same time.");
155
156 if(option_filenames && option_process_only)
157 print_usage(0,NULL,"Cannot use '--process-only' and filenames at the same time.");
158
159 if(!option_filenames && !option_process_only)
160 print_usage(0,NULL,"File names must be specified unless using '--process-only'");
161
162 if(!option_filesort_ramsize)
163 {
164 #if SLIM
165 option_filesort_ramsize=64*1024*1024;
166 #else
167 option_filesort_ramsize=256*1024*1024;
168 #endif
169 }
170 else
171 option_filesort_ramsize*=1024*1024;
172
173 if(!option_tmpdirname)
174 {
175 if(!dirname)
176 option_tmpdirname=".";
177 else
178 option_tmpdirname=dirname;
179 }
180
181 if(!option_process_only)
182 {
183 if(tagging)
184 {
185 if(!ExistsFile(tagging))
186 {
187 fprintf(stderr,"Error: The '--tagging' option specifies a file that does not exist.\n");
188 return(1);
189 }
190 }
191 else
192 {
193 if(ExistsFile(FileName(dirname,prefix,"tagging.xml")))
194 tagging=FileName(dirname,prefix,"tagging.xml");
195 else if(ExistsFile(FileName(DATADIR,NULL,"tagging.xml")))
196 tagging=FileName(DATADIR,NULL,"tagging.xml");
197 else
198 {
199 fprintf(stderr,"Error: The '--tagging' option was not used and the default 'tagging.xml' does not exist.\n");
200 return(1);
201 }
202 }
203
204 if(ParseXMLTaggingRules(tagging))
205 {
206 fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging);
207 return(1);
208 }
209 }
210
211 /* Create new node, segment, way and relation variables */
212
213 OSMNodes=NewNodeList(option_append||option_changes,option_process_only);
214
215 OSMWays=NewWayList(option_append||option_changes,option_process_only);
216
217 OSMRelations=NewRelationList(option_append||option_changes,option_process_only);
218
219 /* Create the error log file */
220
221 if(errorlog)
222 open_errorlog(FileName(dirname,prefix,errorlog),option_append||option_changes||option_process_only,option_keep);
223
224 /* Parse the file */
225
226 if(!option_process_only)
227 {
228 for(arg=1;arg<argc;arg++)
229 {
230 int fd;
231 char *filename,*p;
232
233 if(argv[arg][0]=='-' && argv[arg][1]=='-')
234 continue;
235
236 filename=strcpy(malloc(strlen(argv[arg])+1),argv[arg]);
237
238 fd=ReOpenFile(filename);
239
240 if((p=strstr(filename,".bz2")) && !strcmp(p,".bz2"))
241 {
242 fd=Uncompress_Bzip2(fd);
243 *p=0;
244 }
245
246 if((p=strstr(filename,".gz")) && !strcmp(p,".gz"))
247 {
248 fd=Uncompress_Gzip(fd);
249 *p=0;
250 }
251
252 if(option_changes)
253 {
254 printf("\nParse OSC Data [%s]\n==============\n\n",filename);
255 fflush(stdout);
256
257 if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf"))
258 {
259 logassert(0,"Unable to read a PBF file to apply changes (format does not permit this)");
260 }
261 else if((p=strstr(filename,".o5c")) && !strcmp(p,".o5c"))
262 {
263 if(ParseO5CFile(fd,OSMNodes,OSMWays,OSMRelations))
264 exit(EXIT_FAILURE);
265 }
266 else
267 {
268 if(ParseOSCFile(fd,OSMNodes,OSMWays,OSMRelations))
269 exit(EXIT_FAILURE);
270 }
271 }
272 else
273 {
274 printf("\nParse OSM Data [%s]\n==============\n\n",filename);
275 fflush(stdout);
276
277 if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf"))
278 {
279 if(ParsePBFFile(fd,OSMNodes,OSMWays,OSMRelations))
280 exit(EXIT_FAILURE);
281 }
282 else if((p=strstr(filename,".o5m")) && !strcmp(p,".o5m"))
283 {
284 if(ParseO5MFile(fd,OSMNodes,OSMWays,OSMRelations))
285 exit(EXIT_FAILURE);
286 }
287 else
288 {
289 if(ParseOSMFile(fd,OSMNodes,OSMWays,OSMRelations))
290 exit(EXIT_FAILURE);
291 }
292 }
293
294 CloseFile(fd);
295
296 free(filename);
297 }
298
299 DeleteXMLTaggingRules();
300 }
301
302 FinishNodeList(OSMNodes);
303 FinishWayList(OSMWays);
304 FinishRelationList(OSMRelations);
305
306 if(option_parse_only)
307 {
308 FreeNodeList(OSMNodes,1);
309 FreeWayList(OSMWays,1);
310 FreeRelationList(OSMRelations,1);
311
312 return(0);
313 }
314
315
316 /* Sort the data */
317
318 printf("\nSort OSM Data\n=============\n\n");
319 fflush(stdout);
320
321 /* Sort the nodes, segments, ways and relations */
322
323 SortNodeList(OSMNodes);
324
325 SortWayList(OSMWays);
326
327 SortRelationList(OSMRelations);
328
329 /* Process the data */
330
331 printf("\nProcess OSM Data\n================\n\n");
332 fflush(stdout);
333
334 /* Separate the segments and way names and sort them. */
335
336 OSMSegments=SplitWays(OSMWays,option_keep||option_changes);
337
338 SortWayNames(OSMWays);
339
340 SortSegmentList(OSMSegments);
341
342 /* Remove bad segments (must be after sorting the nodes, segments and ways) */
343
344 RemoveBadSegments(OSMSegments,OSMNodes,OSMWays);
345
346 /* Remove non-highway nodes (must be after removing the bad segments) */
347
348 RemoveNonHighwayNodes(OSMNodes,OSMSegments,option_keep||option_changes);
349
350 /* Process the route relations and first part of turn relations (must be before compacting the ways) */
351
352 ProcessRouteRelations(OSMRelations,OSMWays,option_keep||option_changes);
353
354 ProcessTurnRelations1(OSMRelations,OSMNodes,OSMWays,option_keep||option_changes);
355
356 /* Measure the segments and replace node/way id with index (must be after removing non-highway nodes) */
357
358 MeasureSegments(OSMSegments,OSMNodes,OSMWays);
359
360 /* Index the segments */
361
362 IndexSegments(OSMSegments,OSMNodes,OSMWays);
363
364 /* Convert the turn relations from ways into nodes */
365
366 ProcessTurnRelations2(OSMRelations,OSMNodes,OSMSegments,OSMWays);
367
368 /* Compact the ways (must be after turn relations 2) */
369
370 CompactWayList(OSMWays,OSMSegments);
371
372 /* Index the segments */
373
374 IndexSegments(OSMSegments,OSMNodes,OSMWays);
375
376 /* Prune unwanted nodes/segments. */
377
378 if(option_prune_straight || option_prune_isolated || option_prune_short)
379 {
380 printf("\nPrune Unneeded Data\n===================\n\n");
381 fflush(stdout);
382
383 StartPruning(OSMNodes,OSMSegments,OSMWays);
384
385 if(option_prune_isolated)
386 PruneIsolatedRegions(OSMNodes,OSMSegments,OSMWays,option_prune_isolated);
387
388 if(option_prune_short)
389 PruneShortSegments(OSMNodes,OSMSegments,OSMWays,option_prune_short);
390
391 if(option_prune_straight)
392 PruneStraightHighwayNodes(OSMNodes,OSMSegments,OSMWays,option_prune_straight);
393
394 FinishPruning(OSMNodes,OSMSegments,OSMWays);
395
396 /* Remove the pruned nodes and segments and update the indexes */
397
398 RemovePrunedNodes(OSMNodes,OSMSegments);
399 RemovePrunedSegments(OSMSegments,OSMWays);
400 CompactWayList(OSMWays,OSMSegments);
401 RemovePrunedTurnRelations(OSMRelations,OSMNodes);
402 IndexSegments(OSMSegments,OSMNodes,OSMWays);
403 }
404
405 /* Repeated iteration on Super-Nodes and Super-Segments */
406
407 do
408 {
409 index_t nsuper;
410
411 printf("\nProcess Super-Data (iteration %d)\n================================%s\n\n",iteration,iteration>9?"=":"");
412 fflush(stdout);
413
414 if(iteration==0)
415 {
416 /* Select the super-nodes */
417
418 ChooseSuperNodes(OSMNodes,OSMSegments,OSMWays);
419
420 /* Select the super-segments */
421
422 SuperSegments=CreateSuperSegments(OSMNodes,OSMSegments,OSMWays);
423
424 nsuper=OSMSegments->number;
425 }
426 else
427 {
428 SegmentsX *SuperSegments2;
429
430 /* Select the super-nodes */
431
432 ChooseSuperNodes(OSMNodes,SuperSegments,OSMWays);
433
434 /* Select the super-segments */
435
436 SuperSegments2=CreateSuperSegments(OSMNodes,SuperSegments,OSMWays);
437
438 nsuper=SuperSegments->number;
439
440 FreeSegmentList(SuperSegments);
441
442 SuperSegments=SuperSegments2;
443 }
444
445 /* Sort the super-segments and remove duplicates */
446
447 DeduplicateSuperSegments(SuperSegments,OSMWays);
448
449 /* Index the segments */
450
451 IndexSegments(SuperSegments,OSMNodes,OSMWays);
452
453 /* Check for end condition */
454
455 if(SuperSegments->number==nsuper)
456 quit=1;
457
458 iteration++;
459
460 if(iteration>max_iterations)
461 quit=1;
462 }
463 while(!quit);
464
465 /* Combine the super-segments */
466
467 printf("\nCombine Segments and Super-Segments\n===================================\n\n");
468 fflush(stdout);
469
470 /* Merge the super-segments */
471
472 MergedSegments=MergeSuperSegments(OSMSegments,SuperSegments);
473
474 FreeSegmentList(OSMSegments);
475
476 FreeSegmentList(SuperSegments);
477
478 OSMSegments=MergedSegments;
479
480 /* Re-index the merged segments */
481
482 IndexSegments(OSMSegments,OSMNodes,OSMWays);
483
484 /* Cross reference the nodes and segments */
485
486 printf("\nCross-Reference Nodes and Segments\n==================================\n\n");
487 fflush(stdout);
488
489 /* Sort the nodes and segments geographically */
490
491 SortNodeListGeographically(OSMNodes);
492
493 SortSegmentListGeographically(OSMSegments,OSMNodes);
494
495 /* Re-index the segments */
496
497 IndexSegments(OSMSegments,OSMNodes,OSMWays);
498
499 /* Sort the turn relations geographically */
500
501 SortTurnRelationListGeographically(OSMRelations,OSMNodes,OSMSegments);
502
503 /* Output the results */
504
505 printf("\nWrite Out Database Files\n========================\n\n");
506 fflush(stdout);
507
508 /* Write out the nodes */
509
510 SaveNodeList(OSMNodes,FileName(dirname,prefix,"nodes.mem"),OSMSegments);
511
512 /* Write out the segments */
513
514 SaveSegmentList(OSMSegments,FileName(dirname,prefix,"segments.mem"));
515
516 /* Write out the ways */
517
518 SaveWayList(OSMWays,FileName(dirname,prefix,"ways.mem"));
519
520 /* Write out the relations */
521
522 SaveRelationList(OSMRelations,FileName(dirname,prefix,"relations.mem"));
523
524 /* Close the error log file and process the data */
525
526 if(errorlog)
527 {
528 close_errorlog();
529
530 if(option_keep)
531 {
532 printf("\nCreate Error Log\n================\n\n");
533 fflush(stdout);
534
535 ProcessErrorLogs(OSMNodes,OSMWays,OSMRelations);
536
537 SortErrorLogsGeographically();
538
539 SaveErrorLogs(OSMNodes,FileName(dirname,prefix,"errorlogs.mem"));
540 }
541 }
542
543 /* Free the memory (delete the temporary files) */
544
545 FreeNodeList(OSMNodes,0);
546 FreeWayList(OSMWays,0);
547 FreeRelationList(OSMRelations,0);
548
549 FreeSegmentList(OSMSegments);
550
551 /* Print the total time */
552
553 if(option_logtime)
554 {
555 printf("\n");
556 fprintf_elapsed_time(stdout,&start_time);
557 printf("Complete\n");
558 fflush(stdout);
559 }
560
561 return(0);
562 }
563
564
565 /*++++++++++++++++++++++++++++++++++++++
566 Print out the usage information.
567
568 int detail The level of detail to use - 0 = low, 1 = high.
569
570 const char *argerr The argument that gave the error (if there is one).
571
572 const char *err Other error message (if there is one).
573 ++++++++++++++++++++++++++++++++++++++*/
574
575 static void print_usage(int detail,const char *argerr,const char *err)
576 {
577 fprintf(stderr,
578 "Usage: planetsplitter [--help]\n"
579 " [--dir=<dirname>] [--prefix=<name>]\n"
580 #if defined(USE_PTHREADS) && USE_PTHREADS
581 " [--sort-ram-size=<size>] [--sort-threads=<number>]\n"
582 #else
583 " [--sort-ram-size=<size>]\n"
584 #endif
585 " [--tmpdir=<dirname>]\n"
586 " [--tagging=<filename>]\n"
587 " [--loggable] [--logtime]\n"
588 " [--errorlog[=<name>]]\n"
589 " [--parse-only | --process-only]\n"
590 " [--append] [--keep] [--changes]\n"
591 " [--max-iterations=<number>]\n"
592 " [--prune-none]\n"
593 " [--prune-isolated=<len>]\n"
594 " [--prune-short=<len>]\n"
595 " [--prune-straight=<len>]\n"
596 " [<filename.osm> ... | <filename.osc> ...\n"
597 " | <filename.pbf> ...\n"
598 " | <filename.osm> ... | <filename.osc> ..."
599 #if defined(USE_BZIP2) && USE_BZIP2
600 "\n | <filename.(osm|osc|o5m|o5c).bz2> ..."
601 #endif
602 #if defined(USE_GZIP) && USE_GZIP
603 "\n | <filename.(osm|osc|o5m|o5c).gz> ..."
604 #endif
605 "]\n");
606
607 if(argerr)
608 fprintf(stderr,
609 "\n"
610 "Error with command line parameter: %s\n",argerr);
611
612 if(err)
613 fprintf(stderr,
614 "\n"
615 "Error: %s\n",err);
616
617 if(detail)
618 fprintf(stderr,
619 "\n"
620 "--help Prints this information.\n"
621 "\n"
622 "--dir=<dirname> The directory containing the routing database.\n"
623 "--prefix=<name> The filename prefix for the routing database.\n"
624 "\n"
625 "--sort-ram-size=<size> The amount of RAM (in MB) to use for data sorting\n"
626 #if SLIM
627 " (defaults to 64MB otherwise.)\n"
628 #else
629 " (defaults to 256MB otherwise.)\n"
630 #endif
631 #if defined(USE_PTHREADS) && USE_PTHREADS
632 "--sort-threads=<number> The number of threads to use for data sorting.\n"
633 #endif
634 "\n"
635 "--tmpdir=<dirname> The directory name for temporary files.\n"
636 " (defaults to the '--dir' option directory.)\n"
637 "\n"
638 "--tagging=<filename> The name of the XML file containing the tagging rules\n"
639 " (defaults to 'tagging.xml' with '--dir' and\n"
640 " '--prefix' options or the file installed in\n"
641 " '" DATADIR "').\n"
642 "\n"
643 "--loggable Print progress messages suitable for logging to file.\n"
644 "--logtime Print the elapsed time for each processing step.\n"
645 "--errorlog[=<name>] Log parsing errors to 'error.log' or the given name\n"
646 " (the '--dir' and '--prefix' options are applied).\n"
647 "\n"
648 "--parse-only Parse the OSM/OSC file(s) and store the results.\n"
649 "--process-only Process the stored results from previous option.\n"
650 "--append Parse the OSM file(s) and append to existing results.\n"
651 "--keep Keep the intermediate files after parsing & sorting.\n"
652 "--changes Parse the data as an OSC file and apply the changes.\n"
653 "\n"
654 "--max-iterations=<number> The number of iterations for finding super-nodes\n"
655 " (defaults to 5).\n"
656 "\n"
657 "--prune-none Disable the prune options below, they are re-enabled\n"
658 " by adding them to the command line after this option.\n"
659 "--prune-isolated=<len> Remove access from small disconnected segment groups\n"
660 " (defaults to removing groups under 500m).\n"
661 "--prune-short=<len> Remove short segments (defaults to removing segments\n"
662 " up to a maximum length of 5m).\n"
663 "--prune-straight=<len> Remove nodes in almost straight highways (defaults to\n"
664 " removing nodes up to 3m offset from a straight line).\n"
665 "\n"
666 "<filename.osm>, <filename.osc>, <filename.pbf>, <filename.o5m>, <filename.o5c>\n"
667 " The name(s) of the file(s) to read and parse.\n"
668 " Filenames ending '.pbf' read as PBF, filenames ending\n"
669 " '.o5m' or '.o5c' read as O5M/O5C, others as XML.\n"
670 #if defined(USE_BZIP2) && USE_BZIP2
671 " Filenames ending '.bz2' will be bzip2 uncompressed.\n"
672 #endif
673 #if defined(USE_GZIP) && USE_GZIP
674 " Filenames ending '.gz' will be gzip uncompressed.\n"
675 #endif
676 "\n"
677 "<transport> defaults to all but can be set to:\n"
678 "%s"
679 "\n"
680 "<highway> can be selected from:\n"
681 "%s"
682 "\n"
683 "<property> can be selected from:\n"
684 "%s",
685 TransportList(),HighwayList(),PropertyList());
686
687 exit(!detail);
688 }

Properties

Name Value
cvs:description Planet file splitter.