Routino SVN Repository Browser

Check out the latest version of Routino: svn co http://routino.org/svn/trunk routino

ViewVC logotype

Contents of /trunk/src/planetsplitter.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1350 - (show annotations) (download) (as text)
Thu May 30 16:53:27 2013 UTC (11 years, 9 months ago) by amb
File MIME type: text/x-csrc
File size: 20921 byte(s)
Delete the non-highway nodes by searching for them in the ways rather than
marking them when processing the segments.

1 /***************************************
2 OSM planet file splitter.
3
4 Part of the Routino routing software.
5 ******************/ /******************
6 This file Copyright 2008-2013 Andrew M. Bishop
7
8 This program is free software: you can redistribute it and/or modify
9 it under the terms of the GNU Affero General Public License as published by
10 the Free Software Foundation, either version 3 of the License, or
11 (at your option) any later version.
12
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU Affero General Public License for more details.
17
18 You should have received a copy of the GNU Affero General Public License
19 along with this program. If not, see <http://www.gnu.org/licenses/>.
20 ***************************************/
21
22
23 #include <stdio.h>
24 #include <stdlib.h>
25 #include <unistd.h>
26 #include <string.h>
27 #include <errno.h>
28 #include <sys/time.h>
29
30 #include "types.h"
31 #include "ways.h"
32
33 #include "typesx.h"
34 #include "nodesx.h"
35 #include "segmentsx.h"
36 #include "waysx.h"
37 #include "relationsx.h"
38 #include "superx.h"
39 #include "prunex.h"
40
41 #include "files.h"
42 #include "logging.h"
43 #include "errorlogx.h"
44 #include "functions.h"
45 #include "osmparser.h"
46 #include "tagging.h"
47 #include "uncompress.h"
48
49
50 /* Global variables */
51
52 /*+ The name of the temporary directory. +*/
53 char *option_tmpdirname=NULL;
54
55 /*+ The amount of RAM to use for filesorting. +*/
56 size_t option_filesort_ramsize=0;
57
58 /*+ The number of threads to use for filesorting. +*/
59 int option_filesort_threads=1;
60
61
62 /* Local functions */
63
64 static void print_usage(int detail,const char *argerr,const char *err);
65
66
67 /*++++++++++++++++++++++++++++++++++++++
68 The main program for the planetsplitter.
69 ++++++++++++++++++++++++++++++++++++++*/
70
71 int main(int argc,char** argv)
72 {
73 struct timeval start_time;
74 NodesX *OSMNodes;
75 SegmentsX *OSMSegments,*SuperSegments=NULL,*MergedSegments=NULL;
76 WaysX *OSMWays;
77 RelationsX *OSMRelations;
78 int iteration=0,quit=0;
79 int max_iterations=5;
80 char *dirname=NULL,*prefix=NULL,*tagging=NULL,*errorlog=NULL;
81 int option_parse_only=0,option_process_only=0;
82 int option_append=0,option_keep=0,option_changes=0;
83 int option_filenames=0;
84 int option_prune_isolated=500,option_prune_short=5,option_prune_straight=3;
85 int arg;
86
87 gettimeofday(&start_time,NULL);
88
89 /* Parse the command line arguments */
90
91 for(arg=1;arg<argc;arg++)
92 {
93 if(!strcmp(argv[arg],"--help"))
94 print_usage(1,NULL,NULL);
95 else if(!strncmp(argv[arg],"--dir=",6))
96 dirname=&argv[arg][6];
97 else if(!strncmp(argv[arg],"--prefix=",9))
98 prefix=&argv[arg][9];
99 else if(!strncmp(argv[arg],"--sort-ram-size=",16))
100 option_filesort_ramsize=atoi(&argv[arg][16]);
101 #if defined(USE_PTHREADS) && USE_PTHREADS
102 else if(!strncmp(argv[arg],"--sort-threads=",15))
103 option_filesort_threads=atoi(&argv[arg][15]);
104 #endif
105 else if(!strncmp(argv[arg],"--tmpdir=",9))
106 option_tmpdirname=&argv[arg][9];
107 else if(!strncmp(argv[arg],"--tagging=",10))
108 tagging=&argv[arg][10];
109 else if(!strcmp(argv[arg],"--loggable"))
110 option_loggable=1;
111 else if(!strcmp(argv[arg],"--logtime"))
112 option_logtime=1;
113 else if(!strcmp(argv[arg],"--errorlog"))
114 errorlog="error.log";
115 else if(!strncmp(argv[arg],"--errorlog=",11))
116 errorlog=&argv[arg][11];
117 else if(!strcmp(argv[arg],"--parse-only"))
118 option_parse_only=1;
119 else if(!strcmp(argv[arg],"--process-only"))
120 option_process_only=1;
121 else if(!strcmp(argv[arg],"--append"))
122 option_append=1;
123 else if(!strcmp(argv[arg],"--keep"))
124 option_keep=1;
125 else if(!strcmp(argv[arg],"--changes"))
126 option_changes=1;
127 else if(!strncmp(argv[arg],"--max-iterations=",17))
128 max_iterations=atoi(&argv[arg][17]);
129 else if(!strncmp(argv[arg],"--prune",7))
130 {
131 if(!strcmp(&argv[arg][7],"-none"))
132 option_prune_isolated=option_prune_short=option_prune_straight=0;
133 else if(!strncmp(&argv[arg][7],"-isolated=",10))
134 option_prune_isolated=atoi(&argv[arg][17]);
135 else if(!strncmp(&argv[arg][7],"-short=",7))
136 option_prune_short=atoi(&argv[arg][14]);
137 else if(!strncmp(&argv[arg][7],"-straight=",10))
138 option_prune_straight=atoi(&argv[arg][17]);
139 else
140 print_usage(0,argv[arg],NULL);
141 }
142 else if(argv[arg][0]=='-' && argv[arg][1]=='-')
143 print_usage(0,argv[arg],NULL);
144 else
145 option_filenames++;
146 }
147
148 /* Check the specified command line options */
149
150 if(option_parse_only && option_process_only)
151 print_usage(0,NULL,"Cannot use '--parse-only' and '--process-only' at the same time.");
152
153 if(option_append && option_process_only)
154 print_usage(0,NULL,"Cannot use '--append' and '--process-only' at the same time.");
155
156 if(option_filenames && option_process_only)
157 print_usage(0,NULL,"Cannot use '--process-only' and filenames at the same time.");
158
159 if(!option_filenames && !option_process_only)
160 print_usage(0,NULL,"File names must be specified unless using '--process-only'");
161
162 if(!option_filesort_ramsize)
163 {
164 #if SLIM
165 option_filesort_ramsize=64*1024*1024;
166 #else
167 option_filesort_ramsize=256*1024*1024;
168 #endif
169 }
170 else
171 option_filesort_ramsize*=1024*1024;
172
173 if(!option_tmpdirname)
174 {
175 if(!dirname)
176 option_tmpdirname=".";
177 else
178 option_tmpdirname=dirname;
179 }
180
181 if(!option_process_only)
182 {
183 if(tagging)
184 {
185 if(!ExistsFile(tagging))
186 {
187 fprintf(stderr,"Error: The '--tagging' option specifies a file that does not exist.\n");
188 return(1);
189 }
190 }
191 else
192 {
193 if(ExistsFile(FileName(dirname,prefix,"tagging.xml")))
194 tagging=FileName(dirname,prefix,"tagging.xml");
195 else if(ExistsFile(FileName(DATADIR,NULL,"tagging.xml")))
196 tagging=FileName(DATADIR,NULL,"tagging.xml");
197 else
198 {
199 fprintf(stderr,"Error: The '--tagging' option was not used and the default 'tagging.xml' does not exist.\n");
200 return(1);
201 }
202 }
203
204 if(ParseXMLTaggingRules(tagging))
205 {
206 fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging);
207 return(1);
208 }
209 }
210
211 /* Create new node, segment, way and relation variables */
212
213 OSMNodes=NewNodeList(option_append||option_changes,option_process_only);
214
215 OSMWays=NewWayList(option_append||option_changes,option_process_only);
216
217 OSMRelations=NewRelationList(option_append||option_changes,option_process_only);
218
219 /* Create the error log file */
220
221 if(errorlog)
222 open_errorlog(FileName(dirname,prefix,errorlog),option_append||option_changes||option_process_only,option_keep);
223
224 /* Parse the file */
225
226 if(!option_process_only)
227 {
228 for(arg=1;arg<argc;arg++)
229 {
230 int fd;
231 char *filename,*p;
232
233 if(argv[arg][0]=='-' && argv[arg][1]=='-')
234 continue;
235
236 filename=strcpy(malloc(strlen(argv[arg])+1),argv[arg]);
237
238 fd=ReOpenFile(filename);
239
240 if((p=strstr(filename,".bz2")) && !strcmp(p,".bz2"))
241 {
242 fd=Uncompress_Bzip2(fd);
243 *p=0;
244 }
245
246 if((p=strstr(filename,".gz")) && !strcmp(p,".gz"))
247 {
248 fd=Uncompress_Gzip(fd);
249 *p=0;
250 }
251
252 if(option_changes)
253 {
254 printf("\nParse OSC Data [%s]\n==============\n\n",filename);
255 fflush(stdout);
256
257 if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf"))
258 {
259 logassert(0,"Unable to read a PBF file to apply changes (format does not permit this)");
260 }
261 else if((p=strstr(filename,".o5c")) && !strcmp(p,".o5c"))
262 {
263 if(ParseO5CFile(fd,OSMNodes,OSMWays,OSMRelations))
264 exit(EXIT_FAILURE);
265 }
266 else
267 {
268 if(ParseOSCFile(fd,OSMNodes,OSMWays,OSMRelations))
269 exit(EXIT_FAILURE);
270 }
271 }
272 else
273 {
274 printf("\nParse OSM Data [%s]\n==============\n\n",filename);
275 fflush(stdout);
276
277 if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf"))
278 {
279 if(ParsePBFFile(fd,OSMNodes,OSMWays,OSMRelations))
280 exit(EXIT_FAILURE);
281 }
282 else if((p=strstr(filename,".o5m")) && !strcmp(p,".o5m"))
283 {
284 if(ParseO5MFile(fd,OSMNodes,OSMWays,OSMRelations))
285 exit(EXIT_FAILURE);
286 }
287 else
288 {
289 if(ParseOSMFile(fd,OSMNodes,OSMWays,OSMRelations))
290 exit(EXIT_FAILURE);
291 }
292 }
293
294 CloseFile(fd);
295
296 free(filename);
297 }
298
299 DeleteXMLTaggingRules();
300 }
301
302 FinishNodeList(OSMNodes);
303 FinishWayList(OSMWays);
304 FinishRelationList(OSMRelations);
305
306 if(option_parse_only)
307 {
308 FreeNodeList(OSMNodes,1);
309 FreeWayList(OSMWays,1);
310 FreeRelationList(OSMRelations,1);
311
312 return(0);
313 }
314
315
316 /* Sort the data */
317
318 printf("\nSort OSM Data\n=============\n\n");
319 fflush(stdout);
320
321 /* Sort the nodes, segments, ways and relations */
322
323 SortNodeList(OSMNodes);
324
325 SortWayList(OSMWays);
326
327 SortRelationList(OSMRelations);
328
329 /* Process the data */
330
331 printf("\nProcess OSM Data\n================\n\n");
332 fflush(stdout);
333
334 /* Remove non-highway nodes by looking through the ways */
335
336 RemoveNonHighwayNodes(OSMNodes,OSMWays,option_keep||option_changes);
337
338 /* Separate the segments and way names and sort them. */
339
340 OSMSegments=SplitWays(OSMWays,OSMNodes,option_keep||option_changes);
341
342 SortWayNames(OSMWays);
343
344 SortSegmentList(OSMSegments);
345
346 /* Remove bad segments (must be after sorting the nodes, segments and ways) */
347
348 RemoveBadSegments(OSMSegments,OSMNodes,OSMWays);
349
350 MeasureSegments(OSMSegments,OSMNodes,OSMWays);
351
352 /* Process the route relations and first part of turn relations (must be before compacting the ways) */
353
354 ProcessRouteRelations(OSMRelations,OSMWays,option_keep||option_changes);
355
356 ProcessTurnRelations1(OSMRelations,OSMNodes,OSMWays,option_keep||option_changes);
357
358 /* Index the segments */
359
360 IndexSegments(OSMSegments,OSMNodes,OSMWays);
361
362 /* Convert the turn relations from ways into nodes */
363
364 ProcessTurnRelations2(OSMRelations,OSMNodes,OSMSegments,OSMWays);
365
366 /* Compact the ways (must be after turn relations 2) */
367
368 CompactWayList(OSMWays,OSMSegments);
369
370 /* Index the segments */
371
372 IndexSegments(OSMSegments,OSMNodes,OSMWays);
373
374 /* Prune unwanted nodes/segments. */
375
376 if(option_prune_straight || option_prune_isolated || option_prune_short)
377 {
378 printf("\nPrune Unneeded Data\n===================\n\n");
379 fflush(stdout);
380
381 StartPruning(OSMNodes,OSMSegments,OSMWays);
382
383 if(option_prune_isolated)
384 PruneIsolatedRegions(OSMNodes,OSMSegments,OSMWays,option_prune_isolated);
385
386 if(option_prune_short)
387 PruneShortSegments(OSMNodes,OSMSegments,OSMWays,option_prune_short);
388
389 if(option_prune_straight)
390 PruneStraightHighwayNodes(OSMNodes,OSMSegments,OSMWays,option_prune_straight);
391
392 FinishPruning(OSMNodes,OSMSegments,OSMWays);
393
394 /* Remove the pruned nodes and segments and update the indexes */
395
396 RemovePrunedNodes(OSMNodes,OSMSegments);
397 RemovePrunedSegments(OSMSegments,OSMWays);
398 CompactWayList(OSMWays,OSMSegments);
399 RemovePrunedTurnRelations(OSMRelations,OSMNodes);
400 IndexSegments(OSMSegments,OSMNodes,OSMWays);
401 }
402
403 /* Repeated iteration on Super-Nodes and Super-Segments */
404
405 do
406 {
407 index_t nsuper;
408
409 printf("\nProcess Super-Data (iteration %d)\n================================%s\n\n",iteration,iteration>9?"=":"");
410 fflush(stdout);
411
412 if(iteration==0)
413 {
414 /* Select the super-nodes */
415
416 ChooseSuperNodes(OSMNodes,OSMSegments,OSMWays);
417
418 /* Select the super-segments */
419
420 SuperSegments=CreateSuperSegments(OSMNodes,OSMSegments,OSMWays);
421
422 nsuper=OSMSegments->number;
423 }
424 else
425 {
426 SegmentsX *SuperSegments2;
427
428 /* Select the super-nodes */
429
430 ChooseSuperNodes(OSMNodes,SuperSegments,OSMWays);
431
432 /* Select the super-segments */
433
434 SuperSegments2=CreateSuperSegments(OSMNodes,SuperSegments,OSMWays);
435
436 nsuper=SuperSegments->number;
437
438 FreeSegmentList(SuperSegments);
439
440 SuperSegments=SuperSegments2;
441 }
442
443 /* Sort the super-segments and remove duplicates */
444
445 DeduplicateSuperSegments(SuperSegments,OSMWays);
446
447 /* Index the segments */
448
449 IndexSegments(SuperSegments,OSMNodes,OSMWays);
450
451 /* Check for end condition */
452
453 if(SuperSegments->number==nsuper)
454 quit=1;
455
456 iteration++;
457
458 if(iteration>max_iterations)
459 quit=1;
460 }
461 while(!quit);
462
463 /* Combine the super-segments */
464
465 printf("\nCombine Segments and Super-Segments\n===================================\n\n");
466 fflush(stdout);
467
468 /* Merge the super-segments */
469
470 MergedSegments=MergeSuperSegments(OSMSegments,SuperSegments);
471
472 FreeSegmentList(OSMSegments);
473
474 FreeSegmentList(SuperSegments);
475
476 OSMSegments=MergedSegments;
477
478 /* Re-index the merged segments */
479
480 IndexSegments(OSMSegments,OSMNodes,OSMWays);
481
482 /* Cross reference the nodes and segments */
483
484 printf("\nCross-Reference Nodes and Segments\n==================================\n\n");
485 fflush(stdout);
486
487 /* Sort the nodes and segments geographically */
488
489 SortNodeListGeographically(OSMNodes);
490
491 SortSegmentListGeographically(OSMSegments,OSMNodes);
492
493 /* Re-index the segments */
494
495 IndexSegments(OSMSegments,OSMNodes,OSMWays);
496
497 /* Sort the turn relations geographically */
498
499 SortTurnRelationListGeographically(OSMRelations,OSMNodes,OSMSegments);
500
501 /* Output the results */
502
503 printf("\nWrite Out Database Files\n========================\n\n");
504 fflush(stdout);
505
506 /* Write out the nodes */
507
508 SaveNodeList(OSMNodes,FileName(dirname,prefix,"nodes.mem"),OSMSegments);
509
510 /* Write out the segments */
511
512 SaveSegmentList(OSMSegments,FileName(dirname,prefix,"segments.mem"));
513
514 /* Write out the ways */
515
516 SaveWayList(OSMWays,FileName(dirname,prefix,"ways.mem"));
517
518 /* Write out the relations */
519
520 SaveRelationList(OSMRelations,FileName(dirname,prefix,"relations.mem"));
521
522 /* Close the error log file and process the data */
523
524 if(errorlog)
525 {
526 close_errorlog();
527
528 if(option_keep)
529 {
530 printf("\nCreate Error Log\n================\n\n");
531 fflush(stdout);
532
533 ProcessErrorLogs(OSMNodes,OSMWays,OSMRelations);
534
535 SortErrorLogsGeographically();
536
537 SaveErrorLogs(OSMNodes,FileName(dirname,prefix,"errorlogs.mem"));
538 }
539 }
540
541 /* Free the memory (delete the temporary files) */
542
543 FreeNodeList(OSMNodes,0);
544 FreeWayList(OSMWays,0);
545 FreeRelationList(OSMRelations,0);
546
547 FreeSegmentList(OSMSegments);
548
549 /* Print the total time */
550
551 if(option_logtime)
552 {
553 printf("\n");
554 fprintf_elapsed_time(stdout,&start_time);
555 printf("Complete\n");
556 fflush(stdout);
557 }
558
559 return(0);
560 }
561
562
563 /*++++++++++++++++++++++++++++++++++++++
564 Print out the usage information.
565
566 int detail The level of detail to use - 0 = low, 1 = high.
567
568 const char *argerr The argument that gave the error (if there is one).
569
570 const char *err Other error message (if there is one).
571 ++++++++++++++++++++++++++++++++++++++*/
572
573 static void print_usage(int detail,const char *argerr,const char *err)
574 {
575 fprintf(stderr,
576 "Usage: planetsplitter [--help]\n"
577 " [--dir=<dirname>] [--prefix=<name>]\n"
578 #if defined(USE_PTHREADS) && USE_PTHREADS
579 " [--sort-ram-size=<size>] [--sort-threads=<number>]\n"
580 #else
581 " [--sort-ram-size=<size>]\n"
582 #endif
583 " [--tmpdir=<dirname>]\n"
584 " [--tagging=<filename>]\n"
585 " [--loggable] [--logtime]\n"
586 " [--errorlog[=<name>]]\n"
587 " [--parse-only | --process-only]\n"
588 " [--append] [--keep] [--changes]\n"
589 " [--max-iterations=<number>]\n"
590 " [--prune-none]\n"
591 " [--prune-isolated=<len>]\n"
592 " [--prune-short=<len>]\n"
593 " [--prune-straight=<len>]\n"
594 " [<filename.osm> ... | <filename.osc> ...\n"
595 " | <filename.pbf> ...\n"
596 " | <filename.osm> ... | <filename.osc> ..."
597 #if defined(USE_BZIP2) && USE_BZIP2
598 "\n | <filename.(osm|osc|o5m|o5c).bz2> ..."
599 #endif
600 #if defined(USE_GZIP) && USE_GZIP
601 "\n | <filename.(osm|osc|o5m|o5c).gz> ..."
602 #endif
603 "]\n");
604
605 if(argerr)
606 fprintf(stderr,
607 "\n"
608 "Error with command line parameter: %s\n",argerr);
609
610 if(err)
611 fprintf(stderr,
612 "\n"
613 "Error: %s\n",err);
614
615 if(detail)
616 fprintf(stderr,
617 "\n"
618 "--help Prints this information.\n"
619 "\n"
620 "--dir=<dirname> The directory containing the routing database.\n"
621 "--prefix=<name> The filename prefix for the routing database.\n"
622 "\n"
623 "--sort-ram-size=<size> The amount of RAM (in MB) to use for data sorting\n"
624 #if SLIM
625 " (defaults to 64MB otherwise.)\n"
626 #else
627 " (defaults to 256MB otherwise.)\n"
628 #endif
629 #if defined(USE_PTHREADS) && USE_PTHREADS
630 "--sort-threads=<number> The number of threads to use for data sorting.\n"
631 #endif
632 "\n"
633 "--tmpdir=<dirname> The directory name for temporary files.\n"
634 " (defaults to the '--dir' option directory.)\n"
635 "\n"
636 "--tagging=<filename> The name of the XML file containing the tagging rules\n"
637 " (defaults to 'tagging.xml' with '--dir' and\n"
638 " '--prefix' options or the file installed in\n"
639 " '" DATADIR "').\n"
640 "\n"
641 "--loggable Print progress messages suitable for logging to file.\n"
642 "--logtime Print the elapsed time for each processing step.\n"
643 "--errorlog[=<name>] Log parsing errors to 'error.log' or the given name\n"
644 " (the '--dir' and '--prefix' options are applied).\n"
645 "\n"
646 "--parse-only Parse the OSM/OSC file(s) and store the results.\n"
647 "--process-only Process the stored results from previous option.\n"
648 "--append Parse the OSM file(s) and append to existing results.\n"
649 "--keep Keep the intermediate files after parsing & sorting.\n"
650 "--changes Parse the data as an OSC file and apply the changes.\n"
651 "\n"
652 "--max-iterations=<number> The number of iterations for finding super-nodes\n"
653 " (defaults to 5).\n"
654 "\n"
655 "--prune-none Disable the prune options below, they are re-enabled\n"
656 " by adding them to the command line after this option.\n"
657 "--prune-isolated=<len> Remove access from small disconnected segment groups\n"
658 " (defaults to removing groups under 500m).\n"
659 "--prune-short=<len> Remove short segments (defaults to removing segments\n"
660 " up to a maximum length of 5m).\n"
661 "--prune-straight=<len> Remove nodes in almost straight highways (defaults to\n"
662 " removing nodes up to 3m offset from a straight line).\n"
663 "\n"
664 "<filename.osm>, <filename.osc>, <filename.pbf>, <filename.o5m>, <filename.o5c>\n"
665 " The name(s) of the file(s) to read and parse.\n"
666 " Filenames ending '.pbf' read as PBF, filenames ending\n"
667 " '.o5m' or '.o5c' read as O5M/O5C, others as XML.\n"
668 #if defined(USE_BZIP2) && USE_BZIP2
669 " Filenames ending '.bz2' will be bzip2 uncompressed.\n"
670 #endif
671 #if defined(USE_GZIP) && USE_GZIP
672 " Filenames ending '.gz' will be gzip uncompressed.\n"
673 #endif
674 "\n"
675 "<transport> defaults to all but can be set to:\n"
676 "%s"
677 "\n"
678 "<highway> can be selected from:\n"
679 "%s"
680 "\n"
681 "<property> can be selected from:\n"
682 "%s",
683 TransportList(),HighwayList(),PropertyList());
684
685 exit(!detail);
686 }

Properties

Name Value
cvs:description Planet file splitter.