Routino SVN Repository Browser

Check out the latest version of Routino: svn co http://routino.org/svn/trunk routino

ViewVC logotype

Contents of /trunk/src/planetsplitter.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1352 - (show annotations) (download) (as text)
Thu May 30 17:48:01 2013 UTC (11 years, 10 months ago) by amb
File MIME type: text/x-csrc
File size: 20806 byte(s)
Move the first IndexSegments() function call earlier in the sequence.

1 /***************************************
2 OSM planet file splitter.
3
4 Part of the Routino routing software.
5 ******************/ /******************
6 This file Copyright 2008-2013 Andrew M. Bishop
7
8 This program is free software: you can redistribute it and/or modify
9 it under the terms of the GNU Affero General Public License as published by
10 the Free Software Foundation, either version 3 of the License, or
11 (at your option) any later version.
12
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU Affero General Public License for more details.
17
18 You should have received a copy of the GNU Affero General Public License
19 along with this program. If not, see <http://www.gnu.org/licenses/>.
20 ***************************************/
21
22
23 #include <stdio.h>
24 #include <stdlib.h>
25 #include <unistd.h>
26 #include <string.h>
27 #include <errno.h>
28 #include <sys/time.h>
29
30 #include "types.h"
31 #include "ways.h"
32
33 #include "typesx.h"
34 #include "nodesx.h"
35 #include "segmentsx.h"
36 #include "waysx.h"
37 #include "relationsx.h"
38 #include "superx.h"
39 #include "prunex.h"
40
41 #include "files.h"
42 #include "logging.h"
43 #include "errorlogx.h"
44 #include "functions.h"
45 #include "osmparser.h"
46 #include "tagging.h"
47 #include "uncompress.h"
48
49
50 /* Global variables */
51
52 /*+ The name of the temporary directory. +*/
53 char *option_tmpdirname=NULL;
54
55 /*+ The amount of RAM to use for filesorting. +*/
56 size_t option_filesort_ramsize=0;
57
58 /*+ The number of threads to use for filesorting. +*/
59 int option_filesort_threads=1;
60
61
62 /* Local functions */
63
64 static void print_usage(int detail,const char *argerr,const char *err);
65
66
67 /*++++++++++++++++++++++++++++++++++++++
68 The main program for the planetsplitter.
69 ++++++++++++++++++++++++++++++++++++++*/
70
71 int main(int argc,char** argv)
72 {
73 struct timeval start_time;
74 NodesX *OSMNodes;
75 SegmentsX *OSMSegments,*SuperSegments=NULL,*MergedSegments=NULL;
76 WaysX *OSMWays;
77 RelationsX *OSMRelations;
78 int iteration=0,quit=0;
79 int max_iterations=5;
80 char *dirname=NULL,*prefix=NULL,*tagging=NULL,*errorlog=NULL;
81 int option_parse_only=0,option_process_only=0;
82 int option_append=0,option_keep=0,option_changes=0;
83 int option_filenames=0;
84 int option_prune_isolated=500,option_prune_short=5,option_prune_straight=3;
85 int arg;
86
87 gettimeofday(&start_time,NULL);
88
89 /* Parse the command line arguments */
90
91 for(arg=1;arg<argc;arg++)
92 {
93 if(!strcmp(argv[arg],"--help"))
94 print_usage(1,NULL,NULL);
95 else if(!strncmp(argv[arg],"--dir=",6))
96 dirname=&argv[arg][6];
97 else if(!strncmp(argv[arg],"--prefix=",9))
98 prefix=&argv[arg][9];
99 else if(!strncmp(argv[arg],"--sort-ram-size=",16))
100 option_filesort_ramsize=atoi(&argv[arg][16]);
101 #if defined(USE_PTHREADS) && USE_PTHREADS
102 else if(!strncmp(argv[arg],"--sort-threads=",15))
103 option_filesort_threads=atoi(&argv[arg][15]);
104 #endif
105 else if(!strncmp(argv[arg],"--tmpdir=",9))
106 option_tmpdirname=&argv[arg][9];
107 else if(!strncmp(argv[arg],"--tagging=",10))
108 tagging=&argv[arg][10];
109 else if(!strcmp(argv[arg],"--loggable"))
110 option_loggable=1;
111 else if(!strcmp(argv[arg],"--logtime"))
112 option_logtime=1;
113 else if(!strcmp(argv[arg],"--errorlog"))
114 errorlog="error.log";
115 else if(!strncmp(argv[arg],"--errorlog=",11))
116 errorlog=&argv[arg][11];
117 else if(!strcmp(argv[arg],"--parse-only"))
118 option_parse_only=1;
119 else if(!strcmp(argv[arg],"--process-only"))
120 option_process_only=1;
121 else if(!strcmp(argv[arg],"--append"))
122 option_append=1;
123 else if(!strcmp(argv[arg],"--keep"))
124 option_keep=1;
125 else if(!strcmp(argv[arg],"--changes"))
126 option_changes=1;
127 else if(!strncmp(argv[arg],"--max-iterations=",17))
128 max_iterations=atoi(&argv[arg][17]);
129 else if(!strncmp(argv[arg],"--prune",7))
130 {
131 if(!strcmp(&argv[arg][7],"-none"))
132 option_prune_isolated=option_prune_short=option_prune_straight=0;
133 else if(!strncmp(&argv[arg][7],"-isolated=",10))
134 option_prune_isolated=atoi(&argv[arg][17]);
135 else if(!strncmp(&argv[arg][7],"-short=",7))
136 option_prune_short=atoi(&argv[arg][14]);
137 else if(!strncmp(&argv[arg][7],"-straight=",10))
138 option_prune_straight=atoi(&argv[arg][17]);
139 else
140 print_usage(0,argv[arg],NULL);
141 }
142 else if(argv[arg][0]=='-' && argv[arg][1]=='-')
143 print_usage(0,argv[arg],NULL);
144 else
145 option_filenames++;
146 }
147
148 /* Check the specified command line options */
149
150 if(option_parse_only && option_process_only)
151 print_usage(0,NULL,"Cannot use '--parse-only' and '--process-only' at the same time.");
152
153 if(option_append && option_process_only)
154 print_usage(0,NULL,"Cannot use '--append' and '--process-only' at the same time.");
155
156 if(option_filenames && option_process_only)
157 print_usage(0,NULL,"Cannot use '--process-only' and filenames at the same time.");
158
159 if(!option_filenames && !option_process_only)
160 print_usage(0,NULL,"File names must be specified unless using '--process-only'");
161
162 if(!option_filesort_ramsize)
163 {
164 #if SLIM
165 option_filesort_ramsize=64*1024*1024;
166 #else
167 option_filesort_ramsize=256*1024*1024;
168 #endif
169 }
170 else
171 option_filesort_ramsize*=1024*1024;
172
173 if(!option_tmpdirname)
174 {
175 if(!dirname)
176 option_tmpdirname=".";
177 else
178 option_tmpdirname=dirname;
179 }
180
181 if(!option_process_only)
182 {
183 if(tagging)
184 {
185 if(!ExistsFile(tagging))
186 {
187 fprintf(stderr,"Error: The '--tagging' option specifies a file that does not exist.\n");
188 return(1);
189 }
190 }
191 else
192 {
193 if(ExistsFile(FileName(dirname,prefix,"tagging.xml")))
194 tagging=FileName(dirname,prefix,"tagging.xml");
195 else if(ExistsFile(FileName(DATADIR,NULL,"tagging.xml")))
196 tagging=FileName(DATADIR,NULL,"tagging.xml");
197 else
198 {
199 fprintf(stderr,"Error: The '--tagging' option was not used and the default 'tagging.xml' does not exist.\n");
200 return(1);
201 }
202 }
203
204 if(ParseXMLTaggingRules(tagging))
205 {
206 fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging);
207 return(1);
208 }
209 }
210
211 /* Create new node, segment, way and relation variables */
212
213 OSMNodes=NewNodeList(option_append||option_changes,option_process_only);
214
215 OSMWays=NewWayList(option_append||option_changes,option_process_only);
216
217 OSMRelations=NewRelationList(option_append||option_changes,option_process_only);
218
219 /* Create the error log file */
220
221 if(errorlog)
222 open_errorlog(FileName(dirname,prefix,errorlog),option_append||option_changes||option_process_only,option_keep);
223
224 /* Parse the file */
225
226 if(!option_process_only)
227 {
228 for(arg=1;arg<argc;arg++)
229 {
230 int fd;
231 char *filename,*p;
232
233 if(argv[arg][0]=='-' && argv[arg][1]=='-')
234 continue;
235
236 filename=strcpy(malloc(strlen(argv[arg])+1),argv[arg]);
237
238 fd=ReOpenFile(filename);
239
240 if((p=strstr(filename,".bz2")) && !strcmp(p,".bz2"))
241 {
242 fd=Uncompress_Bzip2(fd);
243 *p=0;
244 }
245
246 if((p=strstr(filename,".gz")) && !strcmp(p,".gz"))
247 {
248 fd=Uncompress_Gzip(fd);
249 *p=0;
250 }
251
252 if(option_changes)
253 {
254 printf("\nParse OSC Data [%s]\n==============\n\n",filename);
255 fflush(stdout);
256
257 if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf"))
258 {
259 logassert(0,"Unable to read a PBF file to apply changes (format does not permit this)");
260 }
261 else if((p=strstr(filename,".o5c")) && !strcmp(p,".o5c"))
262 {
263 if(ParseO5CFile(fd,OSMNodes,OSMWays,OSMRelations))
264 exit(EXIT_FAILURE);
265 }
266 else
267 {
268 if(ParseOSCFile(fd,OSMNodes,OSMWays,OSMRelations))
269 exit(EXIT_FAILURE);
270 }
271 }
272 else
273 {
274 printf("\nParse OSM Data [%s]\n==============\n\n",filename);
275 fflush(stdout);
276
277 if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf"))
278 {
279 if(ParsePBFFile(fd,OSMNodes,OSMWays,OSMRelations))
280 exit(EXIT_FAILURE);
281 }
282 else if((p=strstr(filename,".o5m")) && !strcmp(p,".o5m"))
283 {
284 if(ParseO5MFile(fd,OSMNodes,OSMWays,OSMRelations))
285 exit(EXIT_FAILURE);
286 }
287 else
288 {
289 if(ParseOSMFile(fd,OSMNodes,OSMWays,OSMRelations))
290 exit(EXIT_FAILURE);
291 }
292 }
293
294 CloseFile(fd);
295
296 free(filename);
297 }
298
299 DeleteXMLTaggingRules();
300 }
301
302 FinishNodeList(OSMNodes);
303 FinishWayList(OSMWays);
304 FinishRelationList(OSMRelations);
305
306 if(option_parse_only)
307 {
308 FreeNodeList(OSMNodes,1);
309 FreeWayList(OSMWays,1);
310 FreeRelationList(OSMRelations,1);
311
312 return(0);
313 }
314
315
316 /* Sort the data */
317
318 printf("\nSort OSM Data\n=============\n\n");
319 fflush(stdout);
320
321 /* Sort the nodes, segments, ways and relations */
322
323 SortNodeList(OSMNodes);
324
325 SortWayList(OSMWays);
326
327 SortRelationList(OSMRelations);
328
329 /* Process the data */
330
331 printf("\nProcess OSM Data\n================\n\n");
332 fflush(stdout);
333
334 /* Remove non-highway nodes by looking through the ways */
335
336 RemoveNonHighwayNodes(OSMNodes,OSMWays,option_keep||option_changes);
337
338 /* Separate the segments and way names and sort them. */
339
340 OSMSegments=SplitWays(OSMWays,OSMNodes,option_keep||option_changes);
341
342 SortWayNames(OSMWays);
343
344 SortSegmentList(OSMSegments);
345
346 /* Process the segments and index them */
347
348 ProcessSegments(OSMSegments,OSMNodes,OSMWays);
349
350 IndexSegments(OSMSegments,OSMNodes,OSMWays);
351
352 /* Process the route relations and first part of turn relations (must be before compacting the ways) */
353
354 ProcessRouteRelations(OSMRelations,OSMWays,option_keep||option_changes);
355
356 ProcessTurnRelations1(OSMRelations,OSMNodes,OSMWays,option_keep||option_changes);
357
358 /* Convert the turn relations from ways into nodes */
359
360 ProcessTurnRelations2(OSMRelations,OSMNodes,OSMSegments,OSMWays);
361
362 /* Compact the ways (must be after turn relations 2) */
363
364 CompactWayList(OSMWays,OSMSegments);
365
366 /* Index the segments */
367
368 IndexSegments(OSMSegments,OSMNodes,OSMWays);
369
370 /* Prune unwanted nodes/segments. */
371
372 if(option_prune_straight || option_prune_isolated || option_prune_short)
373 {
374 printf("\nPrune Unneeded Data\n===================\n\n");
375 fflush(stdout);
376
377 StartPruning(OSMNodes,OSMSegments,OSMWays);
378
379 if(option_prune_isolated)
380 PruneIsolatedRegions(OSMNodes,OSMSegments,OSMWays,option_prune_isolated);
381
382 if(option_prune_short)
383 PruneShortSegments(OSMNodes,OSMSegments,OSMWays,option_prune_short);
384
385 if(option_prune_straight)
386 PruneStraightHighwayNodes(OSMNodes,OSMSegments,OSMWays,option_prune_straight);
387
388 FinishPruning(OSMNodes,OSMSegments,OSMWays);
389
390 /* Remove the pruned nodes and segments and update the indexes */
391
392 RemovePrunedNodes(OSMNodes,OSMSegments);
393 RemovePrunedSegments(OSMSegments,OSMWays);
394 CompactWayList(OSMWays,OSMSegments);
395 RemovePrunedTurnRelations(OSMRelations,OSMNodes);
396 IndexSegments(OSMSegments,OSMNodes,OSMWays);
397 }
398
399 /* Repeated iteration on Super-Nodes and Super-Segments */
400
401 do
402 {
403 index_t nsuper;
404
405 printf("\nProcess Super-Data (iteration %d)\n================================%s\n\n",iteration,iteration>9?"=":"");
406 fflush(stdout);
407
408 if(iteration==0)
409 {
410 /* Select the super-nodes */
411
412 ChooseSuperNodes(OSMNodes,OSMSegments,OSMWays);
413
414 /* Select the super-segments */
415
416 SuperSegments=CreateSuperSegments(OSMNodes,OSMSegments,OSMWays);
417
418 nsuper=OSMSegments->number;
419 }
420 else
421 {
422 SegmentsX *SuperSegments2;
423
424 /* Select the super-nodes */
425
426 ChooseSuperNodes(OSMNodes,SuperSegments,OSMWays);
427
428 /* Select the super-segments */
429
430 SuperSegments2=CreateSuperSegments(OSMNodes,SuperSegments,OSMWays);
431
432 nsuper=SuperSegments->number;
433
434 FreeSegmentList(SuperSegments);
435
436 SuperSegments=SuperSegments2;
437 }
438
439 /* Sort the super-segments and remove duplicates */
440
441 DeduplicateSuperSegments(SuperSegments,OSMWays);
442
443 /* Index the segments */
444
445 IndexSegments(SuperSegments,OSMNodes,OSMWays);
446
447 /* Check for end condition */
448
449 if(SuperSegments->number==nsuper)
450 quit=1;
451
452 iteration++;
453
454 if(iteration>max_iterations)
455 quit=1;
456 }
457 while(!quit);
458
459 /* Combine the super-segments */
460
461 printf("\nCombine Segments and Super-Segments\n===================================\n\n");
462 fflush(stdout);
463
464 /* Merge the super-segments */
465
466 MergedSegments=MergeSuperSegments(OSMSegments,SuperSegments);
467
468 FreeSegmentList(OSMSegments);
469
470 FreeSegmentList(SuperSegments);
471
472 OSMSegments=MergedSegments;
473
474 /* Re-index the merged segments */
475
476 IndexSegments(OSMSegments,OSMNodes,OSMWays);
477
478 /* Cross reference the nodes and segments */
479
480 printf("\nCross-Reference Nodes and Segments\n==================================\n\n");
481 fflush(stdout);
482
483 /* Sort the nodes and segments geographically */
484
485 SortNodeListGeographically(OSMNodes);
486
487 SortSegmentListGeographically(OSMSegments,OSMNodes);
488
489 /* Re-index the segments */
490
491 IndexSegments(OSMSegments,OSMNodes,OSMWays);
492
493 /* Sort the turn relations geographically */
494
495 SortTurnRelationListGeographically(OSMRelations,OSMNodes,OSMSegments);
496
497 /* Output the results */
498
499 printf("\nWrite Out Database Files\n========================\n\n");
500 fflush(stdout);
501
502 /* Write out the nodes */
503
504 SaveNodeList(OSMNodes,FileName(dirname,prefix,"nodes.mem"),OSMSegments);
505
506 /* Write out the segments */
507
508 SaveSegmentList(OSMSegments,FileName(dirname,prefix,"segments.mem"));
509
510 /* Write out the ways */
511
512 SaveWayList(OSMWays,FileName(dirname,prefix,"ways.mem"));
513
514 /* Write out the relations */
515
516 SaveRelationList(OSMRelations,FileName(dirname,prefix,"relations.mem"));
517
518 /* Close the error log file and process the data */
519
520 if(errorlog)
521 {
522 close_errorlog();
523
524 if(option_keep)
525 {
526 printf("\nCreate Error Log\n================\n\n");
527 fflush(stdout);
528
529 ProcessErrorLogs(OSMNodes,OSMWays,OSMRelations);
530
531 SortErrorLogsGeographically();
532
533 SaveErrorLogs(OSMNodes,FileName(dirname,prefix,"errorlogs.mem"));
534 }
535 }
536
537 /* Free the memory (delete the temporary files) */
538
539 FreeNodeList(OSMNodes,0);
540 FreeWayList(OSMWays,0);
541 FreeRelationList(OSMRelations,0);
542
543 FreeSegmentList(OSMSegments);
544
545 /* Print the total time */
546
547 if(option_logtime)
548 {
549 printf("\n");
550 fprintf_elapsed_time(stdout,&start_time);
551 printf("Complete\n");
552 fflush(stdout);
553 }
554
555 return(0);
556 }
557
558
559 /*++++++++++++++++++++++++++++++++++++++
560 Print out the usage information.
561
562 int detail The level of detail to use - 0 = low, 1 = high.
563
564 const char *argerr The argument that gave the error (if there is one).
565
566 const char *err Other error message (if there is one).
567 ++++++++++++++++++++++++++++++++++++++*/
568
569 static void print_usage(int detail,const char *argerr,const char *err)
570 {
571 fprintf(stderr,
572 "Usage: planetsplitter [--help]\n"
573 " [--dir=<dirname>] [--prefix=<name>]\n"
574 #if defined(USE_PTHREADS) && USE_PTHREADS
575 " [--sort-ram-size=<size>] [--sort-threads=<number>]\n"
576 #else
577 " [--sort-ram-size=<size>]\n"
578 #endif
579 " [--tmpdir=<dirname>]\n"
580 " [--tagging=<filename>]\n"
581 " [--loggable] [--logtime]\n"
582 " [--errorlog[=<name>]]\n"
583 " [--parse-only | --process-only]\n"
584 " [--append] [--keep] [--changes]\n"
585 " [--max-iterations=<number>]\n"
586 " [--prune-none]\n"
587 " [--prune-isolated=<len>]\n"
588 " [--prune-short=<len>]\n"
589 " [--prune-straight=<len>]\n"
590 " [<filename.osm> ... | <filename.osc> ...\n"
591 " | <filename.pbf> ...\n"
592 " | <filename.osm> ... | <filename.osc> ..."
593 #if defined(USE_BZIP2) && USE_BZIP2
594 "\n | <filename.(osm|osc|o5m|o5c).bz2> ..."
595 #endif
596 #if defined(USE_GZIP) && USE_GZIP
597 "\n | <filename.(osm|osc|o5m|o5c).gz> ..."
598 #endif
599 "]\n");
600
601 if(argerr)
602 fprintf(stderr,
603 "\n"
604 "Error with command line parameter: %s\n",argerr);
605
606 if(err)
607 fprintf(stderr,
608 "\n"
609 "Error: %s\n",err);
610
611 if(detail)
612 fprintf(stderr,
613 "\n"
614 "--help Prints this information.\n"
615 "\n"
616 "--dir=<dirname> The directory containing the routing database.\n"
617 "--prefix=<name> The filename prefix for the routing database.\n"
618 "\n"
619 "--sort-ram-size=<size> The amount of RAM (in MB) to use for data sorting\n"
620 #if SLIM
621 " (defaults to 64MB otherwise.)\n"
622 #else
623 " (defaults to 256MB otherwise.)\n"
624 #endif
625 #if defined(USE_PTHREADS) && USE_PTHREADS
626 "--sort-threads=<number> The number of threads to use for data sorting.\n"
627 #endif
628 "\n"
629 "--tmpdir=<dirname> The directory name for temporary files.\n"
630 " (defaults to the '--dir' option directory.)\n"
631 "\n"
632 "--tagging=<filename> The name of the XML file containing the tagging rules\n"
633 " (defaults to 'tagging.xml' with '--dir' and\n"
634 " '--prefix' options or the file installed in\n"
635 " '" DATADIR "').\n"
636 "\n"
637 "--loggable Print progress messages suitable for logging to file.\n"
638 "--logtime Print the elapsed time for each processing step.\n"
639 "--errorlog[=<name>] Log parsing errors to 'error.log' or the given name\n"
640 " (the '--dir' and '--prefix' options are applied).\n"
641 "\n"
642 "--parse-only Parse the OSM/OSC file(s) and store the results.\n"
643 "--process-only Process the stored results from previous option.\n"
644 "--append Parse the OSM file(s) and append to existing results.\n"
645 "--keep Keep the intermediate files after parsing & sorting.\n"
646 "--changes Parse the data as an OSC file and apply the changes.\n"
647 "\n"
648 "--max-iterations=<number> The number of iterations for finding super-nodes\n"
649 " (defaults to 5).\n"
650 "\n"
651 "--prune-none Disable the prune options below, they are re-enabled\n"
652 " by adding them to the command line after this option.\n"
653 "--prune-isolated=<len> Remove access from small disconnected segment groups\n"
654 " (defaults to removing groups under 500m).\n"
655 "--prune-short=<len> Remove short segments (defaults to removing segments\n"
656 " up to a maximum length of 5m).\n"
657 "--prune-straight=<len> Remove nodes in almost straight highways (defaults to\n"
658 " removing nodes up to 3m offset from a straight line).\n"
659 "\n"
660 "<filename.osm>, <filename.osc>, <filename.pbf>, <filename.o5m>, <filename.o5c>\n"
661 " The name(s) of the file(s) to read and parse.\n"
662 " Filenames ending '.pbf' read as PBF, filenames ending\n"
663 " '.o5m' or '.o5c' read as O5M/O5C, others as XML.\n"
664 #if defined(USE_BZIP2) && USE_BZIP2
665 " Filenames ending '.bz2' will be bzip2 uncompressed.\n"
666 #endif
667 #if defined(USE_GZIP) && USE_GZIP
668 " Filenames ending '.gz' will be gzip uncompressed.\n"
669 #endif
670 "\n"
671 "<transport> defaults to all but can be set to:\n"
672 "%s"
673 "\n"
674 "<highway> can be selected from:\n"
675 "%s"
676 "\n"
677 "<property> can be selected from:\n"
678 "%s",
679 TransportList(),HighwayList(),PropertyList());
680
681 exit(!detail);
682 }

Properties

Name Value
cvs:description Planet file splitter.