Routino SVN Repository Browser

Check out the latest version of Routino: svn co http://routino.org/svn/trunk routino

ViewVC logotype

Contents of /trunk/src/planetsplitter.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1514 - (show annotations) (download) (as text)
Sat Feb 22 14:00:03 2014 UTC (11 years ago) by amb
File MIME type: text/x-csrc
File size: 20938 byte(s)
Add the option for automatic uncompression of .xz compressed files (not enabled
by default in Makefile.conf).

1 /***************************************
2 OSM planet file splitter.
3
4 Part of the Routino routing software.
5 ******************/ /******************
6 This file Copyright 2008-2014 Andrew M. Bishop
7
8 This program is free software: you can redistribute it and/or modify
9 it under the terms of the GNU Affero General Public License as published by
10 the Free Software Foundation, either version 3 of the License, or
11 (at your option) any later version.
12
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU Affero General Public License for more details.
17
18 You should have received a copy of the GNU Affero General Public License
19 along with this program. If not, see <http://www.gnu.org/licenses/>.
20 ***************************************/
21
22
23 #include <stdio.h>
24 #include <stdlib.h>
25 #include <unistd.h>
26 #include <string.h>
27 #include <errno.h>
28
29 #include "types.h"
30 #include "ways.h"
31
32 #include "typesx.h"
33 #include "nodesx.h"
34 #include "segmentsx.h"
35 #include "waysx.h"
36 #include "relationsx.h"
37 #include "superx.h"
38 #include "prunex.h"
39
40 #include "files.h"
41 #include "logging.h"
42 #include "errorlogx.h"
43 #include "functions.h"
44 #include "osmparser.h"
45 #include "tagging.h"
46 #include "uncompress.h"
47
48
49 /* Global variables */
50
51 /*+ The name of the temporary directory. +*/
52 char *option_tmpdirname=NULL;
53
54 /*+ The amount of RAM to use for filesorting. +*/
55 size_t option_filesort_ramsize=0;
56
57 /*+ The number of threads to use for filesorting. +*/
58 int option_filesort_threads=1;
59
60
61 /* Local functions */
62
63 static void print_usage(int detail,const char *argerr,const char *err);
64
65
66 /*++++++++++++++++++++++++++++++++++++++
67 The main program for the planetsplitter.
68 ++++++++++++++++++++++++++++++++++++++*/
69
70 int main(int argc,char** argv)
71 {
72 NodesX *OSMNodes;
73 SegmentsX *OSMSegments,*SuperSegments=NULL,*MergedSegments=NULL;
74 WaysX *OSMWays;
75 RelationsX *OSMRelations;
76 int iteration=0,quit=0;
77 int max_iterations=5;
78 char *dirname=NULL,*prefix=NULL,*tagging=NULL,*errorlog=NULL;
79 int option_parse_only=0,option_process_only=0;
80 int option_append=0,option_keep=0,option_changes=0;
81 int option_filenames=0;
82 int option_prune_isolated=500,option_prune_short=5,option_prune_straight=3;
83 int arg;
84
85 printf_program_start();
86
87 /* Parse the command line arguments */
88
89 for(arg=1;arg<argc;arg++)
90 {
91 if(!strcmp(argv[arg],"--help"))
92 print_usage(1,NULL,NULL);
93 else if(!strncmp(argv[arg],"--dir=",6))
94 dirname=&argv[arg][6];
95 else if(!strncmp(argv[arg],"--prefix=",9))
96 prefix=&argv[arg][9];
97 else if(!strncmp(argv[arg],"--sort-ram-size=",16))
98 option_filesort_ramsize=atoi(&argv[arg][16]);
99 #if defined(USE_PTHREADS) && USE_PTHREADS
100 else if(!strncmp(argv[arg],"--sort-threads=",15))
101 option_filesort_threads=atoi(&argv[arg][15]);
102 #endif
103 else if(!strncmp(argv[arg],"--tmpdir=",9))
104 option_tmpdirname=&argv[arg][9];
105 else if(!strncmp(argv[arg],"--tagging=",10))
106 tagging=&argv[arg][10];
107 else if(!strcmp(argv[arg],"--loggable"))
108 option_loggable=1;
109 else if(!strcmp(argv[arg],"--logtime"))
110 option_logtime=1;
111 else if(!strcmp(argv[arg],"--errorlog"))
112 errorlog="error.log";
113 else if(!strncmp(argv[arg],"--errorlog=",11))
114 errorlog=&argv[arg][11];
115 else if(!strcmp(argv[arg],"--parse-only"))
116 option_parse_only=1;
117 else if(!strcmp(argv[arg],"--process-only"))
118 option_process_only=1;
119 else if(!strcmp(argv[arg],"--append"))
120 option_append=1;
121 else if(!strcmp(argv[arg],"--keep"))
122 option_keep=1;
123 else if(!strcmp(argv[arg],"--changes"))
124 option_changes=1;
125 else if(!strncmp(argv[arg],"--max-iterations=",17))
126 max_iterations=atoi(&argv[arg][17]);
127 else if(!strncmp(argv[arg],"--prune",7))
128 {
129 if(!strcmp(&argv[arg][7],"-none"))
130 option_prune_isolated=option_prune_short=option_prune_straight=0;
131 else if(!strncmp(&argv[arg][7],"-isolated=",10))
132 option_prune_isolated=atoi(&argv[arg][17]);
133 else if(!strncmp(&argv[arg][7],"-short=",7))
134 option_prune_short=atoi(&argv[arg][14]);
135 else if(!strncmp(&argv[arg][7],"-straight=",10))
136 option_prune_straight=atoi(&argv[arg][17]);
137 else
138 print_usage(0,argv[arg],NULL);
139 }
140 else if(argv[arg][0]=='-' && argv[arg][1]=='-')
141 print_usage(0,argv[arg],NULL);
142 else
143 option_filenames++;
144 }
145
146 /* Check the specified command line options */
147
148 if(option_parse_only && option_process_only)
149 print_usage(0,NULL,"Cannot use '--parse-only' and '--process-only' at the same time.");
150
151 if(option_append && option_process_only)
152 print_usage(0,NULL,"Cannot use '--append' and '--process-only' at the same time.");
153
154 if(option_filenames && option_process_only)
155 print_usage(0,NULL,"Cannot use '--process-only' and filenames at the same time.");
156
157 if(!option_filenames && !option_process_only)
158 print_usage(0,NULL,"File names must be specified unless using '--process-only'");
159
160 if(!option_filesort_ramsize)
161 {
162 #if SLIM
163 option_filesort_ramsize=64*1024*1024;
164 #else
165 option_filesort_ramsize=256*1024*1024;
166 #endif
167 }
168 else
169 option_filesort_ramsize*=1024*1024;
170
171 if(!option_tmpdirname)
172 {
173 if(!dirname)
174 option_tmpdirname=".";
175 else
176 option_tmpdirname=dirname;
177 }
178
179 if(!option_process_only)
180 {
181 if(tagging)
182 {
183 if(!ExistsFile(tagging))
184 {
185 fprintf(stderr,"Error: The '--tagging' option specifies a file that does not exist.\n");
186 exit(EXIT_FAILURE);
187 }
188 }
189 else
190 {
191 if(ExistsFile(FileName(dirname,prefix,"tagging.xml")))
192 tagging=FileName(dirname,prefix,"tagging.xml");
193 else if(ExistsFile(FileName(DATADIR,NULL,"tagging.xml")))
194 tagging=FileName(DATADIR,NULL,"tagging.xml");
195 else
196 {
197 fprintf(stderr,"Error: The '--tagging' option was not used and the default 'tagging.xml' does not exist.\n");
198 exit(EXIT_FAILURE);
199 }
200 }
201
202 if(ParseXMLTaggingRules(tagging))
203 {
204 fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging);
205 exit(EXIT_FAILURE);
206 }
207 }
208
209 /* Create new node, segment, way and relation variables */
210
211 OSMNodes=NewNodeList(option_append||option_changes,option_process_only);
212
213 OSMWays=NewWayList(option_append||option_changes,option_process_only);
214
215 OSMRelations=NewRelationList(option_append||option_changes,option_process_only);
216
217 /* Create the error log file */
218
219 if(errorlog)
220 open_errorlog(FileName(dirname,prefix,errorlog),option_append||option_changes||option_process_only,option_keep);
221
222 /* Parse the file */
223
224 if(!option_process_only)
225 {
226 for(arg=1;arg<argc;arg++)
227 {
228 int fd;
229 char *filename,*p;
230
231 if(argv[arg][0]=='-' && argv[arg][1]=='-')
232 continue;
233
234 filename=strcpy(malloc(strlen(argv[arg])+1),argv[arg]);
235
236 fd=OpenFile(filename);
237
238 if((p=strstr(filename,".bz2")) && !strcmp(p,".bz2"))
239 {
240 fd=Uncompress_Bzip2(fd);
241 *p=0;
242 }
243
244 if((p=strstr(filename,".gz")) && !strcmp(p,".gz"))
245 {
246 fd=Uncompress_Gzip(fd);
247 *p=0;
248 }
249
250 if((p=strstr(filename,".xz")) && !strcmp(p,".xz"))
251 {
252 fd=Uncompress_Xz(fd);
253 *p=0;
254 }
255
256 if(option_changes)
257 {
258 printf("\nParse OSC Data [%s]\n==============\n\n",filename);
259 fflush(stdout);
260
261 if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf"))
262 {
263 logassert(0,"Unable to read a PBF file to apply changes (format does not permit this)");
264 }
265 else if((p=strstr(filename,".o5c")) && !strcmp(p,".o5c"))
266 {
267 if(ParseO5CFile(fd,OSMNodes,OSMWays,OSMRelations))
268 exit(EXIT_FAILURE);
269 }
270 else
271 {
272 if(ParseOSCFile(fd,OSMNodes,OSMWays,OSMRelations))
273 exit(EXIT_FAILURE);
274 }
275 }
276 else
277 {
278 printf("\nParse OSM Data [%s]\n==============\n\n",filename);
279 fflush(stdout);
280
281 if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf"))
282 {
283 if(ParsePBFFile(fd,OSMNodes,OSMWays,OSMRelations))
284 exit(EXIT_FAILURE);
285 }
286 else if((p=strstr(filename,".o5m")) && !strcmp(p,".o5m"))
287 {
288 if(ParseO5MFile(fd,OSMNodes,OSMWays,OSMRelations))
289 exit(EXIT_FAILURE);
290 }
291 else
292 {
293 if(ParseOSMFile(fd,OSMNodes,OSMWays,OSMRelations))
294 exit(EXIT_FAILURE);
295 }
296 }
297
298 CloseFile(fd);
299
300 free(filename);
301 }
302
303 DeleteXMLTaggingRules();
304 }
305
306 FinishNodeList(OSMNodes);
307 FinishWayList(OSMWays);
308 FinishRelationList(OSMRelations);
309
310 if(option_parse_only)
311 {
312 FreeNodeList(OSMNodes,1);
313 FreeWayList(OSMWays,1);
314 FreeRelationList(OSMRelations,1);
315
316 return(0);
317 }
318
319
320 /* Sort the data */
321
322 printf("\nSort OSM Data\n=============\n\n");
323 fflush(stdout);
324
325 /* Sort the nodes, ways and relations */
326
327 SortNodeList(OSMNodes);
328
329 SortWayList(OSMWays);
330
331 SortRelationList(OSMRelations);
332
333 /* Process the data */
334
335 printf("\nProcess OSM Data\n================\n\n");
336 fflush(stdout);
337
338 /* Remove non-highway nodes by looking through the ways */
339
340 RemoveNonHighwayNodes(OSMNodes,OSMWays,option_keep||option_changes);
341
342 /* Separate the segments and way names and sort them. */
343
344 OSMSegments=SplitWays(OSMWays,OSMNodes,option_keep||option_changes);
345
346 SortWayNames(OSMWays);
347
348 SortSegmentList(OSMSegments);
349
350 /* Process the segments and index them */
351
352 ProcessSegments(OSMSegments,OSMNodes,OSMWays);
353
354 IndexSegments(OSMSegments,OSMNodes,OSMWays);
355
356 /* Process the route relations and turn relations (must be before compacting the ways) */
357
358 ProcessRouteRelations(OSMRelations,OSMWays,option_keep||option_changes);
359
360 ProcessTurnRelations(OSMRelations,OSMNodes,OSMSegments,OSMWays,option_keep||option_changes);
361
362 /* Compact the ways (must be after processing turn relations) */
363
364 CompactWayList(OSMWays,OSMSegments);
365
366 /* Index the segments */
367
368 IndexSegments(OSMSegments,OSMNodes,OSMWays);
369
370 /* Prune unwanted nodes/segments. */
371
372 if(option_prune_straight || option_prune_isolated || option_prune_short)
373 {
374 printf("\nPrune Unneeded Data\n===================\n\n");
375 fflush(stdout);
376
377 StartPruning(OSMNodes,OSMSegments,OSMWays);
378
379 if(option_prune_straight)
380 PruneStraightHighwayNodes(OSMNodes,OSMSegments,OSMWays,option_prune_straight);
381
382 if(option_prune_isolated)
383 PruneIsolatedRegions(OSMNodes,OSMSegments,OSMWays,option_prune_isolated);
384
385 if(option_prune_short)
386 PruneShortSegments(OSMNodes,OSMSegments,OSMWays,option_prune_short);
387
388 FinishPruning(OSMNodes,OSMSegments,OSMWays);
389
390 /* Remove the pruned nodes, segments, ways and relations and update the indexes */
391
392 RemovePrunedNodes(OSMNodes,OSMSegments);
393 RemovePrunedSegments(OSMSegments,OSMWays);
394 CompactWayList(OSMWays,OSMSegments);
395 RemovePrunedTurnRelations(OSMRelations,OSMNodes);
396
397 IndexSegments(OSMSegments,OSMNodes,OSMWays);
398 }
399
400 /* Repeated iteration on Super-Nodes and Super-Segments */
401
402 do
403 {
404 index_t nsuper;
405
406 printf("\nProcess Super-Data (iteration %d)\n================================%s\n\n",iteration,iteration>9?"=":"");
407 fflush(stdout);
408
409 if(iteration==0)
410 {
411 /* Select the super-nodes */
412
413 ChooseSuperNodes(OSMNodes,OSMSegments,OSMWays);
414
415 /* Select the super-segments */
416
417 SuperSegments=CreateSuperSegments(OSMNodes,OSMSegments,OSMWays);
418
419 nsuper=OSMSegments->number;
420 }
421 else
422 {
423 SegmentsX *SuperSegments2;
424
425 /* Select the super-nodes */
426
427 ChooseSuperNodes(OSMNodes,SuperSegments,OSMWays);
428
429 /* Select the super-segments */
430
431 SuperSegments2=CreateSuperSegments(OSMNodes,SuperSegments,OSMWays);
432
433 nsuper=SuperSegments->number;
434
435 FreeSegmentList(SuperSegments);
436
437 SuperSegments=SuperSegments2;
438 }
439
440 /* Sort the super-segments and remove duplicates */
441
442 DeduplicateSuperSegments(SuperSegments,OSMWays);
443
444 /* Index the segments */
445
446 IndexSegments(SuperSegments,OSMNodes,OSMWays);
447
448 /* Check for end condition */
449
450 if(SuperSegments->number==nsuper)
451 quit=1;
452
453 iteration++;
454
455 if(iteration>max_iterations)
456 quit=1;
457 }
458 while(!quit);
459
460 /* Combine the super-segments */
461
462 printf("\nCombine Segments and Super-Segments\n===================================\n\n");
463 fflush(stdout);
464
465 /* Merge the super-segments */
466
467 MergedSegments=MergeSuperSegments(OSMSegments,SuperSegments);
468
469 FreeSegmentList(OSMSegments);
470
471 FreeSegmentList(SuperSegments);
472
473 OSMSegments=MergedSegments;
474
475 /* Cross reference the nodes and segments */
476
477 printf("\nCross-Reference Nodes and Segments\n==================================\n\n");
478 fflush(stdout);
479
480 /* Sort the nodes and segments geographically */
481
482 SortNodeListGeographically(OSMNodes);
483
484 SortSegmentListGeographically(OSMSegments,OSMNodes);
485
486 /* Re-index the segments */
487
488 IndexSegments(OSMSegments,OSMNodes,OSMWays);
489
490 /* Sort the turn relations geographically */
491
492 SortTurnRelationListGeographically(OSMRelations,OSMNodes,OSMSegments);
493
494 /* Output the results */
495
496 printf("\nWrite Out Database Files\n========================\n\n");
497 fflush(stdout);
498
499 /* Write out the nodes */
500
501 SaveNodeList(OSMNodes,FileName(dirname,prefix,"nodes.mem"),OSMSegments);
502
503 /* Write out the segments */
504
505 SaveSegmentList(OSMSegments,FileName(dirname,prefix,"segments.mem"));
506
507 /* Write out the ways */
508
509 SaveWayList(OSMWays,FileName(dirname,prefix,"ways.mem"));
510
511 /* Write out the relations */
512
513 SaveRelationList(OSMRelations,FileName(dirname,prefix,"relations.mem"));
514
515 /* Close the error log file and process the data */
516
517 if(errorlog)
518 {
519 close_errorlog();
520
521 if(option_keep)
522 {
523 ErrorLogsX *OSMErrorLogs;
524
525 printf("\nCreate Error Log\n================\n\n");
526 fflush(stdout);
527
528 OSMErrorLogs=NewErrorLogList();
529
530 ProcessErrorLogs(OSMErrorLogs,OSMNodes,OSMWays,OSMRelations);
531
532 SortErrorLogsGeographically(OSMErrorLogs);
533
534 SaveErrorLogs(OSMErrorLogs,FileName(dirname,prefix,"errorlogs.mem"));
535
536 FreeErrorLogList(OSMErrorLogs);
537 }
538 }
539
540 /* Free the memory (delete the temporary files) */
541
542 FreeNodeList(OSMNodes,0);
543 FreeWayList(OSMWays,0);
544 FreeRelationList(OSMRelations,0);
545
546 FreeSegmentList(OSMSegments);
547
548 printf_program_end();
549
550 return(0);
551 }
552
553
554 /*++++++++++++++++++++++++++++++++++++++
555 Print out the usage information.
556
557 int detail The level of detail to use - 0 = low, 1 = high.
558
559 const char *argerr The argument that gave the error (if there is one).
560
561 const char *err Other error message (if there is one).
562 ++++++++++++++++++++++++++++++++++++++*/
563
564 static void print_usage(int detail,const char *argerr,const char *err)
565 {
566 fprintf(stderr,
567 "Usage: planetsplitter [--help]\n"
568 " [--dir=<dirname>] [--prefix=<name>]\n"
569 #if defined(USE_PTHREADS) && USE_PTHREADS
570 " [--sort-ram-size=<size>] [--sort-threads=<number>]\n"
571 #else
572 " [--sort-ram-size=<size>]\n"
573 #endif
574 " [--tmpdir=<dirname>]\n"
575 " [--tagging=<filename>]\n"
576 " [--loggable] [--logtime]\n"
577 " [--errorlog[=<name>]]\n"
578 " [--parse-only | --process-only]\n"
579 " [--append] [--keep] [--changes]\n"
580 " [--max-iterations=<number>]\n"
581 " [--prune-none]\n"
582 " [--prune-isolated=<len>]\n"
583 " [--prune-short=<len>]\n"
584 " [--prune-straight=<len>]\n"
585 " [<filename.osm> ... | <filename.osc> ...\n"
586 " | <filename.pbf> ...\n"
587 " | <filename.o5m> ... | <filename.o5c> ..."
588 #if defined(USE_BZIP2) && USE_BZIP2
589 "\n | <filename.(osm|osc|o5m|o5c).bz2> ..."
590 #endif
591 #if defined(USE_GZIP) && USE_GZIP
592 "\n | <filename.(osm|osc|o5m|o5c).gz> ..."
593 #endif
594 #if defined(USE_XZ) && USE_XZ
595 "\n | <filename.(osm|osc|o5m|o5c).xz> ..."
596 #endif
597 "]\n");
598
599 if(argerr)
600 fprintf(stderr,
601 "\n"
602 "Error with command line parameter: %s\n",argerr);
603
604 if(err)
605 fprintf(stderr,
606 "\n"
607 "Error: %s\n",err);
608
609 if(detail)
610 fprintf(stderr,
611 "\n"
612 "--help Prints this information.\n"
613 "\n"
614 "--dir=<dirname> The directory containing the routing database.\n"
615 "--prefix=<name> The filename prefix for the routing database.\n"
616 "\n"
617 "--sort-ram-size=<size> The amount of RAM (in MB) to use for data sorting\n"
618 #if SLIM
619 " (defaults to 64MB otherwise.)\n"
620 #else
621 " (defaults to 256MB otherwise.)\n"
622 #endif
623 #if defined(USE_PTHREADS) && USE_PTHREADS
624 "--sort-threads=<number> The number of threads to use for data sorting.\n"
625 #endif
626 "\n"
627 "--tmpdir=<dirname> The directory name for temporary files.\n"
628 " (defaults to the '--dir' option directory.)\n"
629 "\n"
630 "--tagging=<filename> The name of the XML file containing the tagging rules\n"
631 " (defaults to 'tagging.xml' with '--dir' and\n"
632 " '--prefix' options or the file installed in\n"
633 " '" DATADIR "').\n"
634 "\n"
635 "--loggable Print progress messages suitable for logging to file.\n"
636 "--logtime Print the elapsed time for each processing step.\n"
637 "--errorlog[=<name>] Log parsing errors to 'error.log' or the given name\n"
638 " (the '--dir' and '--prefix' options are applied).\n"
639 "\n"
640 "--parse-only Parse the OSM/OSC file(s) and store the results.\n"
641 "--process-only Process the stored results from previous option.\n"
642 "--append Parse the OSM file(s) and append to existing results.\n"
643 "--keep Keep the intermediate files after parsing & sorting.\n"
644 "--changes Parse the data as an OSC file and apply the changes.\n"
645 "\n"
646 "--max-iterations=<number> The number of iterations for finding super-nodes\n"
647 " (defaults to 5).\n"
648 "\n"
649 "--prune-none Disable the prune options below, they are re-enabled\n"
650 " by adding them to the command line after this option.\n"
651 "--prune-isolated=<len> Remove access from small disconnected segment groups\n"
652 " (defaults to removing groups under 500m).\n"
653 "--prune-short=<len> Remove short segments (defaults to removing segments\n"
654 " up to a maximum length of 5m).\n"
655 "--prune-straight=<len> Remove nodes in almost straight highways (defaults to\n"
656 " removing nodes up to 3m offset from a straight line).\n"
657 "\n"
658 "<filename.osm>, <filename.osc>, <filename.pbf>, <filename.o5m>, <filename.o5c>\n"
659 " The name(s) of the file(s) to read and parse.\n"
660 " Filenames ending '.pbf' read as PBF, filenames ending\n"
661 " '.o5m' or '.o5c' read as O5M/O5C, others as XML.\n"
662 #if defined(USE_BZIP2) && USE_BZIP2
663 " Filenames ending '.bz2' will be bzip2 uncompressed.\n"
664 #endif
665 #if defined(USE_GZIP) && USE_GZIP
666 " Filenames ending '.gz' will be gzip uncompressed.\n"
667 #endif
668 #if defined(USE_XZ) && USE_XZ
669 " Filenames ending '.xz' will be xz uncompressed.\n"
670 #endif
671 "\n"
672 "<transport> defaults to all but can be set to:\n"
673 "%s"
674 "\n"
675 "<highway> can be selected from:\n"
676 "%s"
677 "\n"
678 "<property> can be selected from:\n"
679 "%s",
680 TransportList(),HighwayList(),PropertyList());
681
682 exit(!detail);
683 }

Properties

Name Value
cvs:description Planet file splitter.