Routino SVN Repository Browser

Check out the latest version of Routino: svn co http://routino.org/svn/trunk routino

ViewVC logotype

Contents of /trunk/src/planetsplitter.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1428 - (show annotations) (download) (as text)
Thu Jun 27 18:29:33 2013 UTC (11 years, 8 months ago) by amb
File MIME type: text/x-csrc
File size: 20477 byte(s)
Put the next1 pointer in the segmentx object rather than in-memory.

1 /***************************************
2 OSM planet file splitter.
3
4 Part of the Routino routing software.
5 ******************/ /******************
6 This file Copyright 2008-2013 Andrew M. Bishop
7
8 This program is free software: you can redistribute it and/or modify
9 it under the terms of the GNU Affero General Public License as published by
10 the Free Software Foundation, either version 3 of the License, or
11 (at your option) any later version.
12
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU Affero General Public License for more details.
17
18 You should have received a copy of the GNU Affero General Public License
19 along with this program. If not, see <http://www.gnu.org/licenses/>.
20 ***************************************/
21
22
23 #include <stdio.h>
24 #include <stdlib.h>
25 #include <unistd.h>
26 #include <string.h>
27 #include <errno.h>
28
29 #include "types.h"
30 #include "ways.h"
31
32 #include "typesx.h"
33 #include "nodesx.h"
34 #include "segmentsx.h"
35 #include "waysx.h"
36 #include "relationsx.h"
37 #include "superx.h"
38 #include "prunex.h"
39
40 #include "files.h"
41 #include "logging.h"
42 #include "errorlogx.h"
43 #include "functions.h"
44 #include "osmparser.h"
45 #include "tagging.h"
46 #include "uncompress.h"
47
48
49 /* Global variables */
50
51 /*+ The name of the temporary directory. +*/
52 char *option_tmpdirname=NULL;
53
54 /*+ The amount of RAM to use for filesorting. +*/
55 size_t option_filesort_ramsize=0;
56
57 /*+ The number of threads to use for filesorting. +*/
58 int option_filesort_threads=1;
59
60
61 /* Local functions */
62
63 static void print_usage(int detail,const char *argerr,const char *err);
64
65
66 /*++++++++++++++++++++++++++++++++++++++
67 The main program for the planetsplitter.
68 ++++++++++++++++++++++++++++++++++++++*/
69
70 int main(int argc,char** argv)
71 {
72 NodesX *OSMNodes;
73 SegmentsX *OSMSegments,*SuperSegments=NULL,*MergedSegments=NULL;
74 WaysX *OSMWays;
75 RelationsX *OSMRelations;
76 int iteration=0,quit=0;
77 int max_iterations=5;
78 char *dirname=NULL,*prefix=NULL,*tagging=NULL,*errorlog=NULL;
79 int option_parse_only=0,option_process_only=0;
80 int option_append=0,option_keep=0,option_changes=0;
81 int option_filenames=0;
82 int option_prune_isolated=500,option_prune_short=5,option_prune_straight=3;
83 int arg;
84
85 printf_program_start();
86
87 /* Parse the command line arguments */
88
89 for(arg=1;arg<argc;arg++)
90 {
91 if(!strcmp(argv[arg],"--help"))
92 print_usage(1,NULL,NULL);
93 else if(!strncmp(argv[arg],"--dir=",6))
94 dirname=&argv[arg][6];
95 else if(!strncmp(argv[arg],"--prefix=",9))
96 prefix=&argv[arg][9];
97 else if(!strncmp(argv[arg],"--sort-ram-size=",16))
98 option_filesort_ramsize=atoi(&argv[arg][16]);
99 #if defined(USE_PTHREADS) && USE_PTHREADS
100 else if(!strncmp(argv[arg],"--sort-threads=",15))
101 option_filesort_threads=atoi(&argv[arg][15]);
102 #endif
103 else if(!strncmp(argv[arg],"--tmpdir=",9))
104 option_tmpdirname=&argv[arg][9];
105 else if(!strncmp(argv[arg],"--tagging=",10))
106 tagging=&argv[arg][10];
107 else if(!strcmp(argv[arg],"--loggable"))
108 option_loggable=1;
109 else if(!strcmp(argv[arg],"--logtime"))
110 option_logtime=1;
111 else if(!strcmp(argv[arg],"--errorlog"))
112 errorlog="error.log";
113 else if(!strncmp(argv[arg],"--errorlog=",11))
114 errorlog=&argv[arg][11];
115 else if(!strcmp(argv[arg],"--parse-only"))
116 option_parse_only=1;
117 else if(!strcmp(argv[arg],"--process-only"))
118 option_process_only=1;
119 else if(!strcmp(argv[arg],"--append"))
120 option_append=1;
121 else if(!strcmp(argv[arg],"--keep"))
122 option_keep=1;
123 else if(!strcmp(argv[arg],"--changes"))
124 option_changes=1;
125 else if(!strncmp(argv[arg],"--max-iterations=",17))
126 max_iterations=atoi(&argv[arg][17]);
127 else if(!strncmp(argv[arg],"--prune",7))
128 {
129 if(!strcmp(&argv[arg][7],"-none"))
130 option_prune_isolated=option_prune_short=option_prune_straight=0;
131 else if(!strncmp(&argv[arg][7],"-isolated=",10))
132 option_prune_isolated=atoi(&argv[arg][17]);
133 else if(!strncmp(&argv[arg][7],"-short=",7))
134 option_prune_short=atoi(&argv[arg][14]);
135 else if(!strncmp(&argv[arg][7],"-straight=",10))
136 option_prune_straight=atoi(&argv[arg][17]);
137 else
138 print_usage(0,argv[arg],NULL);
139 }
140 else if(argv[arg][0]=='-' && argv[arg][1]=='-')
141 print_usage(0,argv[arg],NULL);
142 else
143 option_filenames++;
144 }
145
146 /* Check the specified command line options */
147
148 if(option_parse_only && option_process_only)
149 print_usage(0,NULL,"Cannot use '--parse-only' and '--process-only' at the same time.");
150
151 if(option_append && option_process_only)
152 print_usage(0,NULL,"Cannot use '--append' and '--process-only' at the same time.");
153
154 if(option_filenames && option_process_only)
155 print_usage(0,NULL,"Cannot use '--process-only' and filenames at the same time.");
156
157 if(!option_filenames && !option_process_only)
158 print_usage(0,NULL,"File names must be specified unless using '--process-only'");
159
160 if(!option_filesort_ramsize)
161 {
162 #if SLIM
163 option_filesort_ramsize=64*1024*1024;
164 #else
165 option_filesort_ramsize=256*1024*1024;
166 #endif
167 }
168 else
169 option_filesort_ramsize*=1024*1024;
170
171 if(!option_tmpdirname)
172 {
173 if(!dirname)
174 option_tmpdirname=".";
175 else
176 option_tmpdirname=dirname;
177 }
178
179 if(!option_process_only)
180 {
181 if(tagging)
182 {
183 if(!ExistsFile(tagging))
184 {
185 fprintf(stderr,"Error: The '--tagging' option specifies a file that does not exist.\n");
186 exit(EXIT_FAILURE);
187 }
188 }
189 else
190 {
191 if(ExistsFile(FileName(dirname,prefix,"tagging.xml")))
192 tagging=FileName(dirname,prefix,"tagging.xml");
193 else if(ExistsFile(FileName(DATADIR,NULL,"tagging.xml")))
194 tagging=FileName(DATADIR,NULL,"tagging.xml");
195 else
196 {
197 fprintf(stderr,"Error: The '--tagging' option was not used and the default 'tagging.xml' does not exist.\n");
198 exit(EXIT_FAILURE);
199 }
200 }
201
202 if(ParseXMLTaggingRules(tagging))
203 {
204 fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging);
205 exit(EXIT_FAILURE);
206 }
207 }
208
209 /* Create new node, segment, way and relation variables */
210
211 OSMNodes=NewNodeList(option_append||option_changes,option_process_only);
212
213 OSMWays=NewWayList(option_append||option_changes,option_process_only);
214
215 OSMRelations=NewRelationList(option_append||option_changes,option_process_only);
216
217 /* Create the error log file */
218
219 if(errorlog)
220 open_errorlog(FileName(dirname,prefix,errorlog),option_append||option_changes||option_process_only,option_keep);
221
222 /* Parse the file */
223
224 if(!option_process_only)
225 {
226 for(arg=1;arg<argc;arg++)
227 {
228 int fd;
229 char *filename,*p;
230
231 if(argv[arg][0]=='-' && argv[arg][1]=='-')
232 continue;
233
234 filename=strcpy(malloc(strlen(argv[arg])+1),argv[arg]);
235
236 fd=OpenFile(filename);
237
238 if((p=strstr(filename,".bz2")) && !strcmp(p,".bz2"))
239 {
240 fd=Uncompress_Bzip2(fd);
241 *p=0;
242 }
243
244 if((p=strstr(filename,".gz")) && !strcmp(p,".gz"))
245 {
246 fd=Uncompress_Gzip(fd);
247 *p=0;
248 }
249
250 if(option_changes)
251 {
252 printf("\nParse OSC Data [%s]\n==============\n\n",filename);
253 fflush(stdout);
254
255 if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf"))
256 {
257 logassert(0,"Unable to read a PBF file to apply changes (format does not permit this)");
258 }
259 else if((p=strstr(filename,".o5c")) && !strcmp(p,".o5c"))
260 {
261 if(ParseO5CFile(fd,OSMNodes,OSMWays,OSMRelations))
262 exit(EXIT_FAILURE);
263 }
264 else
265 {
266 if(ParseOSCFile(fd,OSMNodes,OSMWays,OSMRelations))
267 exit(EXIT_FAILURE);
268 }
269 }
270 else
271 {
272 printf("\nParse OSM Data [%s]\n==============\n\n",filename);
273 fflush(stdout);
274
275 if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf"))
276 {
277 if(ParsePBFFile(fd,OSMNodes,OSMWays,OSMRelations))
278 exit(EXIT_FAILURE);
279 }
280 else if((p=strstr(filename,".o5m")) && !strcmp(p,".o5m"))
281 {
282 if(ParseO5MFile(fd,OSMNodes,OSMWays,OSMRelations))
283 exit(EXIT_FAILURE);
284 }
285 else
286 {
287 if(ParseOSMFile(fd,OSMNodes,OSMWays,OSMRelations))
288 exit(EXIT_FAILURE);
289 }
290 }
291
292 CloseFile(fd);
293
294 free(filename);
295 }
296
297 DeleteXMLTaggingRules();
298 }
299
300 FinishNodeList(OSMNodes);
301 FinishWayList(OSMWays);
302 FinishRelationList(OSMRelations);
303
304 if(option_parse_only)
305 {
306 FreeNodeList(OSMNodes,1);
307 FreeWayList(OSMWays,1);
308 FreeRelationList(OSMRelations,1);
309
310 return(0);
311 }
312
313
314 /* Sort the data */
315
316 printf("\nSort OSM Data\n=============\n\n");
317 fflush(stdout);
318
319 /* Sort the nodes, ways and relations */
320
321 SortNodeList(OSMNodes);
322
323 SortWayList(OSMWays);
324
325 SortRelationList(OSMRelations);
326
327 /* Process the data */
328
329 printf("\nProcess OSM Data\n================\n\n");
330 fflush(stdout);
331
332 /* Remove non-highway nodes by looking through the ways */
333
334 RemoveNonHighwayNodes(OSMNodes,OSMWays,option_keep||option_changes);
335
336 /* Separate the segments and way names and sort them. */
337
338 OSMSegments=SplitWays(OSMWays,OSMNodes,option_keep||option_changes);
339
340 SortWayNames(OSMWays);
341
342 SortSegmentList(OSMSegments);
343
344 /* Process the segments and index them */
345
346 ProcessSegments(OSMSegments,OSMNodes,OSMWays);
347
348 IndexSegments(OSMSegments,OSMNodes,OSMWays);
349
350 /* Process the route relations and turn relations (must be before compacting the ways) */
351
352 ProcessRouteRelations(OSMRelations,OSMWays,option_keep||option_changes);
353
354 ProcessTurnRelations(OSMRelations,OSMNodes,OSMSegments,OSMWays,option_keep||option_changes);
355
356 /* Compact the ways (must be after processing turn relations) */
357
358 CompactWayList(OSMWays,OSMSegments);
359
360 /* Index the segments */
361
362 IndexSegments(OSMSegments,OSMNodes,OSMWays);
363
364 /* Prune unwanted nodes/segments. */
365
366 if(option_prune_straight || option_prune_isolated || option_prune_short)
367 {
368 printf("\nPrune Unneeded Data\n===================\n\n");
369 fflush(stdout);
370
371 if(option_prune_isolated)
372 PruneIsolatedRegions(OSMNodes,OSMSegments,OSMWays,option_prune_isolated);
373
374 if(option_prune_short)
375 PruneShortSegments(OSMNodes,OSMSegments,OSMWays,option_prune_short);
376
377 if(option_prune_straight)
378 PruneStraightHighwayNodes(OSMNodes,OSMSegments,OSMWays,option_prune_straight);
379
380 /* Remove the pruned nodes, segments, ways and relations and update the indexes */
381
382 RemovePrunedNodes(OSMNodes,OSMSegments);
383 RemovePrunedSegments(OSMSegments,OSMWays);
384 CompactWayList(OSMWays,OSMSegments);
385 RemovePrunedTurnRelations(OSMRelations,OSMNodes);
386
387 IndexSegments(OSMSegments,OSMNodes,OSMWays);
388 }
389
390 /* Repeated iteration on Super-Nodes and Super-Segments */
391
392 do
393 {
394 index_t nsuper;
395
396 printf("\nProcess Super-Data (iteration %d)\n================================%s\n\n",iteration,iteration>9?"=":"");
397 fflush(stdout);
398
399 if(iteration==0)
400 {
401 /* Select the super-nodes */
402
403 ChooseSuperNodes(OSMNodes,OSMSegments,OSMWays);
404
405 /* Select the super-segments */
406
407 SuperSegments=CreateSuperSegments(OSMNodes,OSMSegments,OSMWays);
408
409 nsuper=OSMSegments->number;
410 }
411 else
412 {
413 SegmentsX *SuperSegments2;
414
415 /* Select the super-nodes */
416
417 ChooseSuperNodes(OSMNodes,SuperSegments,OSMWays);
418
419 /* Select the super-segments */
420
421 SuperSegments2=CreateSuperSegments(OSMNodes,SuperSegments,OSMWays);
422
423 nsuper=SuperSegments->number;
424
425 FreeSegmentList(SuperSegments);
426
427 SuperSegments=SuperSegments2;
428 }
429
430 /* Sort the super-segments and remove duplicates */
431
432 DeduplicateSuperSegments(SuperSegments,OSMWays);
433
434 /* Index the segments */
435
436 IndexSegments(SuperSegments,OSMNodes,OSMWays);
437
438 /* Check for end condition */
439
440 if(SuperSegments->number==nsuper)
441 quit=1;
442
443 iteration++;
444
445 if(iteration>max_iterations)
446 quit=1;
447 }
448 while(!quit);
449
450 /* Combine the super-segments */
451
452 printf("\nCombine Segments and Super-Segments\n===================================\n\n");
453 fflush(stdout);
454
455 /* Merge the super-segments */
456
457 MergedSegments=MergeSuperSegments(OSMSegments,SuperSegments);
458
459 FreeSegmentList(OSMSegments);
460
461 FreeSegmentList(SuperSegments);
462
463 OSMSegments=MergedSegments;
464
465 /* Cross reference the nodes and segments */
466
467 printf("\nCross-Reference Nodes and Segments\n==================================\n\n");
468 fflush(stdout);
469
470 /* Sort the nodes and segments geographically */
471
472 SortNodeListGeographically(OSMNodes);
473
474 SortSegmentListGeographically(OSMSegments,OSMNodes);
475
476 /* Re-index the segments */
477
478 IndexSegments(OSMSegments,OSMNodes,OSMWays);
479
480 /* Sort the turn relations geographically */
481
482 SortTurnRelationListGeographically(OSMRelations,OSMNodes,OSMSegments);
483
484 /* Output the results */
485
486 printf("\nWrite Out Database Files\n========================\n\n");
487 fflush(stdout);
488
489 /* Write out the nodes */
490
491 SaveNodeList(OSMNodes,FileName(dirname,prefix,"nodes.mem"),OSMSegments);
492
493 /* Write out the segments */
494
495 SaveSegmentList(OSMSegments,FileName(dirname,prefix,"segments.mem"));
496
497 /* Write out the ways */
498
499 SaveWayList(OSMWays,FileName(dirname,prefix,"ways.mem"));
500
501 /* Write out the relations */
502
503 SaveRelationList(OSMRelations,FileName(dirname,prefix,"relations.mem"));
504
505 /* Close the error log file and process the data */
506
507 if(errorlog)
508 {
509 close_errorlog();
510
511 if(option_keep)
512 {
513 ErrorLogsX *OSMErrorLogs;
514
515 printf("\nCreate Error Log\n================\n\n");
516 fflush(stdout);
517
518 OSMErrorLogs=NewErrorLogList();
519
520 ProcessErrorLogs(OSMErrorLogs,OSMNodes,OSMWays,OSMRelations);
521
522 SortErrorLogsGeographically(OSMErrorLogs);
523
524 SaveErrorLogs(OSMErrorLogs,FileName(dirname,prefix,"errorlogs.mem"));
525
526 FreeErrorLogList(OSMErrorLogs);
527 }
528 }
529
530 /* Free the memory (delete the temporary files) */
531
532 FreeNodeList(OSMNodes,0);
533 FreeWayList(OSMWays,0);
534 FreeRelationList(OSMRelations,0);
535
536 FreeSegmentList(OSMSegments);
537
538 printf_program_end();
539
540 return(0);
541 }
542
543
544 /*++++++++++++++++++++++++++++++++++++++
545 Print out the usage information.
546
547 int detail The level of detail to use - 0 = low, 1 = high.
548
549 const char *argerr The argument that gave the error (if there is one).
550
551 const char *err Other error message (if there is one).
552 ++++++++++++++++++++++++++++++++++++++*/
553
554 static void print_usage(int detail,const char *argerr,const char *err)
555 {
556 fprintf(stderr,
557 "Usage: planetsplitter [--help]\n"
558 " [--dir=<dirname>] [--prefix=<name>]\n"
559 #if defined(USE_PTHREADS) && USE_PTHREADS
560 " [--sort-ram-size=<size>] [--sort-threads=<number>]\n"
561 #else
562 " [--sort-ram-size=<size>]\n"
563 #endif
564 " [--tmpdir=<dirname>]\n"
565 " [--tagging=<filename>]\n"
566 " [--loggable] [--logtime]\n"
567 " [--errorlog[=<name>]]\n"
568 " [--parse-only | --process-only]\n"
569 " [--append] [--keep] [--changes]\n"
570 " [--max-iterations=<number>]\n"
571 " [--prune-none]\n"
572 " [--prune-isolated=<len>]\n"
573 " [--prune-short=<len>]\n"
574 " [--prune-straight=<len>]\n"
575 " [<filename.osm> ... | <filename.osc> ...\n"
576 " | <filename.pbf> ...\n"
577 " | <filename.o5m> ... | <filename.o5c> ..."
578 #if defined(USE_BZIP2) && USE_BZIP2
579 "\n | <filename.(osm|osc|o5m|o5c).bz2> ..."
580 #endif
581 #if defined(USE_GZIP) && USE_GZIP
582 "\n | <filename.(osm|osc|o5m|o5c).gz> ..."
583 #endif
584 "]\n");
585
586 if(argerr)
587 fprintf(stderr,
588 "\n"
589 "Error with command line parameter: %s\n",argerr);
590
591 if(err)
592 fprintf(stderr,
593 "\n"
594 "Error: %s\n",err);
595
596 if(detail)
597 fprintf(stderr,
598 "\n"
599 "--help Prints this information.\n"
600 "\n"
601 "--dir=<dirname> The directory containing the routing database.\n"
602 "--prefix=<name> The filename prefix for the routing database.\n"
603 "\n"
604 "--sort-ram-size=<size> The amount of RAM (in MB) to use for data sorting\n"
605 #if SLIM
606 " (defaults to 64MB otherwise.)\n"
607 #else
608 " (defaults to 256MB otherwise.)\n"
609 #endif
610 #if defined(USE_PTHREADS) && USE_PTHREADS
611 "--sort-threads=<number> The number of threads to use for data sorting.\n"
612 #endif
613 "\n"
614 "--tmpdir=<dirname> The directory name for temporary files.\n"
615 " (defaults to the '--dir' option directory.)\n"
616 "\n"
617 "--tagging=<filename> The name of the XML file containing the tagging rules\n"
618 " (defaults to 'tagging.xml' with '--dir' and\n"
619 " '--prefix' options or the file installed in\n"
620 " '" DATADIR "').\n"
621 "\n"
622 "--loggable Print progress messages suitable for logging to file.\n"
623 "--logtime Print the elapsed time for each processing step.\n"
624 "--errorlog[=<name>] Log parsing errors to 'error.log' or the given name\n"
625 " (the '--dir' and '--prefix' options are applied).\n"
626 "\n"
627 "--parse-only Parse the OSM/OSC file(s) and store the results.\n"
628 "--process-only Process the stored results from previous option.\n"
629 "--append Parse the OSM file(s) and append to existing results.\n"
630 "--keep Keep the intermediate files after parsing & sorting.\n"
631 "--changes Parse the data as an OSC file and apply the changes.\n"
632 "\n"
633 "--max-iterations=<number> The number of iterations for finding super-nodes\n"
634 " (defaults to 5).\n"
635 "\n"
636 "--prune-none Disable the prune options below, they are re-enabled\n"
637 " by adding them to the command line after this option.\n"
638 "--prune-isolated=<len> Remove access from small disconnected segment groups\n"
639 " (defaults to removing groups under 500m).\n"
640 "--prune-short=<len> Remove short segments (defaults to removing segments\n"
641 " up to a maximum length of 5m).\n"
642 "--prune-straight=<len> Remove nodes in almost straight highways (defaults to\n"
643 " removing nodes up to 3m offset from a straight line).\n"
644 "\n"
645 "<filename.osm>, <filename.osc>, <filename.pbf>, <filename.o5m>, <filename.o5c>\n"
646 " The name(s) of the file(s) to read and parse.\n"
647 " Filenames ending '.pbf' read as PBF, filenames ending\n"
648 " '.o5m' or '.o5c' read as O5M/O5C, others as XML.\n"
649 #if defined(USE_BZIP2) && USE_BZIP2
650 " Filenames ending '.bz2' will be bzip2 uncompressed.\n"
651 #endif
652 #if defined(USE_GZIP) && USE_GZIP
653 " Filenames ending '.gz' will be gzip uncompressed.\n"
654 #endif
655 "\n"
656 "<transport> defaults to all but can be set to:\n"
657 "%s"
658 "\n"
659 "<highway> can be selected from:\n"
660 "%s"
661 "\n"
662 "<property> can be selected from:\n"
663 "%s",
664 TransportList(),HighwayList(),PropertyList());
665
666 exit(!detail);
667 }

Properties

Name Value
cvs:description Planet file splitter.