Routino SVN Repository Browser

Check out the latest version of Routino: svn co http://routino.org/svn/trunk routino

/[routino]/trunk/src/planetsplitter.c
ViewVC logotype

Contents of /trunk/src/planetsplitter.c

Parent Directory Parent Directory | Revision Log Revision Log

Revision 1194 - (show annotations) (download) (as text)
Thu Dec 13 19:18:28 2012 UTC (12 years, 3 months ago) by amb
File MIME type: text/x-csrc
File size: 18968 byte(s) 
Add the ability to read bzip2 compressed files when specified by name.
1 /***************************************
2 OSM planet file splitter.
3
4 Part of the Routino routing software.
5 ******************/ /******************
6 This file Copyright 2008-2012 Andrew M. Bishop
7
8 This program is free software: you can redistribute it and/or modify
9 it under the terms of the GNU Affero General Public License as published by
10 the Free Software Foundation, either version 3 of the License, or
11 (at your option) any later version.
12
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU Affero General Public License for more details.
17
18 You should have received a copy of the GNU Affero General Public License
19 along with this program. If not, see <http://www.gnu.org/licenses/>.
20 ***************************************/
21
22
23 #include <stdio.h>
24 #include <stdlib.h>
25 #include <unistd.h>
26 #include <string.h>
27 #include <errno.h>
28 #include <sys/time.h>
29
30 #include "types.h"
31 #include "ways.h"
32
33 #include "typesx.h"
34 #include "nodesx.h"
35 #include "segmentsx.h"
36 #include "waysx.h"
37 #include "relationsx.h"
38 #include "superx.h"
39 #include "prunex.h"
40
41 #include "files.h"
42 #include "logging.h"
43 #include "functions.h"
44 #include "osmparser.h"
45 #include "tagging.h"
46 #include "uncompress.h"
47
48
49 /* Global variables */
50
51 /*+ The name of the temporary directory. +*/
52 char *option_tmpdirname=NULL;
53
54 /*+ The amount of RAM to use for filesorting. +*/
55 size_t option_filesort_ramsize=0;
56
57 /*+ The number of threads to use for filesorting. +*/
58 int option_filesort_threads=1;
59
60
61 /* Local functions */
62
63 static void print_usage(int detail,const char *argerr,const char *err);
64
65
66 /*++++++++++++++++++++++++++++++++++++++
67 The main program for the planetsplitter.
68 ++++++++++++++++++++++++++++++++++++++*/
69
70 int main(int argc,char** argv)
71 {
72 struct timeval start_time;
73 NodesX *Nodes;
74 SegmentsX *Segments,*SuperSegments=NULL,*MergedSegments=NULL;
75 WaysX *Ways;
76 RelationsX *Relations;
77 int iteration=0,quit=0;
78 int max_iterations=5;
79 char *dirname=NULL,*prefix=NULL,*tagging=NULL,*errorlog=NULL;
80 int option_parse_only=0,option_process_only=0;
81 int option_append=0,option_keep=0,option_changes=0;
82 int option_filenames=0;
83 int option_prune_isolated=500,option_prune_short=5,option_prune_straight=3;
84 int arg;
85
86 gettimeofday(&start_time,NULL);
87
88 /* Parse the command line arguments */
89
90 for(arg=1;arg<argc;arg++)
91 {
92 if(!strcmp(argv[arg],"--help"))
93 print_usage(1,NULL,NULL);
94 else if(!strncmp(argv[arg],"--sort-ram-size=",16))
95 option_filesort_ramsize=atoi(&argv[arg][16]);
96 #if defined(USE_PTHREADS) && USE_PTHREADS
97 else if(!strncmp(argv[arg],"--sort-threads=",15))
98 option_filesort_threads=atoi(&argv[arg][15]);
99 #endif
100 else if(!strncmp(argv[arg],"--dir=",6))
101 dirname=&argv[arg][6];
102 else if(!strncmp(argv[arg],"--tmpdir=",9))
103 option_tmpdirname=&argv[arg][9];
104 else if(!strncmp(argv[arg],"--prefix=",9))
105 prefix=&argv[arg][9];
106 else if(!strcmp(argv[arg],"--parse-only"))
107 option_parse_only=1;
108 else if(!strcmp(argv[arg],"--process-only"))
109 option_process_only=1;
110 else if(!strcmp(argv[arg],"--append"))
111 option_append=1;
112 else if(!strcmp(argv[arg],"--keep"))
113 option_keep=1;
114 else if(!strcmp(argv[arg],"--changes"))
115 option_changes=1;
116 else if(!strcmp(argv[arg],"--loggable"))
117 option_loggable=1;
118 else if(!strcmp(argv[arg],"--logtime"))
119 option_logtime=1;
120 else if(!strcmp(argv[arg],"--errorlog"))
121 errorlog="error.log";
122 else if(!strncmp(argv[arg],"--errorlog=",11))
123 errorlog=&argv[arg][11];
124 else if(!strncmp(argv[arg],"--max-iterations=",17))
125 max_iterations=atoi(&argv[arg][17]);
126 else if(!strncmp(argv[arg],"--tagging=",10))
127 tagging=&argv[arg][10];
128 else if(!strncmp(argv[arg],"--prune",7))
129 {
130 if(!strcmp(&argv[arg][7],"-none"))
131 option_prune_isolated=option_prune_short=option_prune_straight=0;
132 else if(!strncmp(&argv[arg][7],"-isolated=",10))
133 option_prune_isolated=atoi(&argv[arg][17]);
134 else if(!strncmp(&argv[arg][7],"-short=",7))
135 option_prune_short=atoi(&argv[arg][14]);
136 else if(!strncmp(&argv[arg][7],"-straight=",10))
137 option_prune_straight=atoi(&argv[arg][17]);
138 else
139 print_usage(0,argv[arg],NULL);
140 }
141 else if(argv[arg][0]=='-' && argv[arg][1]=='-')
142 print_usage(0,argv[arg],NULL);
143 else
144 option_filenames++;
145 }
146
147 /* Check the specified command line options */
148
149 if(option_parse_only && option_process_only)
150 print_usage(0,NULL,"Cannot use '--parse-only' and '--process-only' at the same time.");
151
152 if(option_append && option_process_only)
153 print_usage(0,NULL,"Cannot use '--append' and '--process-only' at the same time.");
154
155 if(option_filenames && option_process_only)
156 print_usage(0,NULL,"Cannot use '--process-only' and filenames at the same time.");
157
158 if(!option_filesort_ramsize)
159 {
160 #if SLIM
161 option_filesort_ramsize=64*1024*1024;
162 #else
163 option_filesort_ramsize=256*1024*1024;
164 #endif
165 }
166 else
167 option_filesort_ramsize*=1024*1024;
168
169 if(!option_tmpdirname)
170 {
171 if(!dirname)
172 option_tmpdirname=".";
173 else
174 option_tmpdirname=dirname;
175 }
176
177 if(!option_process_only)
178 {
179 if(tagging)
180 {
181 if(!ExistsFile(tagging))
182 {
183 fprintf(stderr,"Error: The '--tagging' option specifies a file that does not exist.\n");
184 return(1);
185 }
186 }
187 else
188 {
189 if(ExistsFile(FileName(dirname,prefix,"tagging.xml")))
190 tagging=FileName(dirname,prefix,"tagging.xml");
191 else if(ExistsFile(FileName(DATADIR,NULL,"tagging.xml")))
192 tagging=FileName(DATADIR,NULL,"tagging.xml");
193 else
194 {
195 fprintf(stderr,"Error: The '--tagging' option was not used and the default 'tagging.xml' does not exist.\n");
196 return(1);
197 }
198 }
199
200 if(ParseXMLTaggingRules(tagging))
201 {
202 fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging);
203 return(1);
204 }
205 }
206
207 /* Create new node, segment, way and relation variables */
208
209 Nodes=NewNodeList(option_append||option_changes,option_process_only);
210
211 Segments=NewSegmentList(option_append||option_changes,option_process_only);
212
213 Ways=NewWayList(option_append||option_changes,option_process_only);
214
215 Relations=NewRelationList(option_append||option_changes,option_process_only);
216
217 /* Create the error log file */
218
219 if(errorlog)
220 open_errorlog(FileName(dirname,prefix,errorlog),option_append||option_changes||option_process_only);
221
222 /* Parse the file */
223
224 if(!option_process_only)
225 {
226 if(option_filenames)
227 {
228 for(arg=1;arg<argc;arg++)
229 {
230 int fd;
231 char *p;
232
233 if(argv[arg][0]=='-' && argv[arg][1]=='-')
234 continue;
235
236 fd=ReOpenFile(argv[arg]);
237
238 if((p=strstr(argv[arg],".bz2")) && !strcmp(p,".bz2"))
239 fd=Uncompress_Bzip2(fd);
240
241 if(option_changes)
242 {
243 printf("\nParse OSC Data [%s]\n==============\n\n",argv[arg]);
244 fflush(stdout);
245
246 if(ParseOSC(fd,Nodes,Segments,Ways,Relations))
247 exit(EXIT_FAILURE);
248 }
249 else
250 {
251 printf("\nParse OSM Data [%s]\n==============\n\n",argv[arg]);
252 fflush(stdout);
253
254 if(ParseOSM(fd,Nodes,Segments,Ways,Relations))
255 exit(EXIT_FAILURE);
256 }
257
258 CloseFile(fd);
259 }
260 }
261 else
262 {
263 if(option_changes)
264 {
265 printf("\nParse OSC Data\n==============\n\n");
266 fflush(stdout);
267
268 if(ParseOSC(STDIN_FILENO,Nodes,Segments,Ways,Relations))
269 exit(EXIT_FAILURE);
270 }
271 else
272 {
273 printf("\nParse OSM Data\n==============\n\n");
274 fflush(stdout);
275
276 if(ParseOSM(STDIN_FILENO,Nodes,Segments,Ways,Relations))
277 exit(EXIT_FAILURE);
278 }
279 }
280
281 DeleteXMLTaggingRules();
282 }
283
284 FinishNodeList(Nodes);
285 FinishSegmentList(Segments);
286 FinishWayList(Ways);
287 FinishRelationList(Relations);
288
289 if(option_parse_only)
290 {
291 FreeNodeList(Nodes,1);
292 FreeSegmentList(Segments,1);
293 FreeWayList(Ways,1);
294 FreeRelationList(Relations,1);
295
296 return(0);
297 }
298
299
300 /* Sort the data */
301
302 printf("\nSort OSM Data\n=============\n\n");
303 fflush(stdout);
304
305 /* Sort the nodes, segments, ways and relations */
306
307 SortNodeList(Nodes);
308
309 if(option_changes)
310 ApplySegmentChanges(Segments);
311
312 SortSegmentList(Segments);
313
314 SortWayList(Ways);
315
316 SortRelationList(Relations);
317
318 /* Process the data */
319
320 printf("\nProcess OSM Data\n================\n\n");
321 fflush(stdout);
322
323 /* Extract the way names (must be before using the ways) */
324
325 ExtractWayNames(Ways,option_keep||option_changes);
326
327 /* Remove bad segments (must be after sorting the nodes, segments and ways) */
328
329 RemoveBadSegments(Segments,Nodes,Ways,option_keep||option_changes);
330
331 /* Remove non-highway nodes (must be after removing the bad segments) */
332
333 RemoveNonHighwayNodes(Nodes,Segments,option_keep||option_changes);
334
335 /* Process the route relations and first part of turn relations (must be before compacting the ways) */
336
337 ProcessRouteRelations(Relations,Ways,option_keep||option_changes);
338
339 ProcessTurnRelations1(Relations,Nodes,Ways,option_keep||option_changes);
340
341 /* Measure the segments and replace node/way id with index (must be after removing non-highway nodes) */
342
343 MeasureSegments(Segments,Nodes,Ways);
344
345 /* Index the segments */
346
347 IndexSegments(Segments,Nodes,Ways);
348
349 /* Convert the turn relations from ways into nodes */
350
351 ProcessTurnRelations2(Relations,Nodes,Segments,Ways);
352
353 /* Compact the ways (must be after turn relations 2) */
354
355 CompactWayList(Ways,Segments);
356
357 /* Index the segments */
358
359 IndexSegments(Segments,Nodes,Ways);
360
361 /* Prune unwanted nodes/segments. */
362
363 if(option_prune_straight || option_prune_isolated || option_prune_short)
364 {
365 printf("\nPrune Unneeded Data\n===================\n\n");
366 fflush(stdout);
367
368 StartPruning(Nodes,Segments,Ways);
369
370 if(option_prune_straight)
371 PruneStraightHighwayNodes(Nodes,Segments,Ways,option_prune_straight);
372
373 if(option_prune_isolated)
374 PruneIsolatedRegions(Nodes,Segments,Ways,option_prune_isolated);
375
376 if(option_prune_short)
377 PruneShortSegments(Nodes,Segments,Ways,option_prune_short);
378
379 FinishPruning(Nodes,Segments,Ways);
380
381 /* Remove the pruned nodes and segments and update the indexes */
382
383 RemovePrunedNodes(Nodes,Segments);
384 RemovePrunedSegments(Segments,Ways);
385 CompactWayList(Ways,Segments);
386 RemovePrunedTurnRelations(Relations,Nodes);
387 IndexSegments(Segments,Nodes,Ways);
388 }
389
390 /* Repeated iteration on Super-Nodes and Super-Segments */
391
392 do
393 {
394 int nsuper;
395
396 printf("\nProcess Super-Data (iteration %d)\n================================%s\n\n",iteration,iteration>9?"=":"");
397 fflush(stdout);
398
399 if(iteration==0)
400 {
401 /* Select the super-nodes */
402
403 ChooseSuperNodes(Nodes,Segments,Ways);
404
405 /* Select the super-segments */
406
407 SuperSegments=CreateSuperSegments(Nodes,Segments,Ways);
408
409 nsuper=Segments->number;
410 }
411 else
412 {
413 SegmentsX *SuperSegments2;
414
415 /* Select the super-nodes */
416
417 ChooseSuperNodes(Nodes,SuperSegments,Ways);
418
419 /* Select the super-segments */
420
421 SuperSegments2=CreateSuperSegments(Nodes,SuperSegments,Ways);
422
423 nsuper=SuperSegments->number;
424
425 FreeSegmentList(SuperSegments,0);
426
427 SuperSegments=SuperSegments2;
428 }
429
430 /* Sort the super-segments and remove duplicates */
431
432 DeduplicateSuperSegments(SuperSegments,Ways);
433
434 /* Index the segments */
435
436 IndexSegments(SuperSegments,Nodes,Ways);
437
438 /* Check for end condition */
439
440 if(SuperSegments->number==nsuper)
441 quit=1;
442
443 iteration++;
444
445 if(iteration>max_iterations)
446 quit=1;
447 }
448 while(!quit);
449
450 /* Combine the super-segments */
451
452 printf("\nCombine Segments and Super-Segments\n===================================\n\n");
453 fflush(stdout);
454
455 /* Merge the super-segments */
456
457 MergedSegments=MergeSuperSegments(Segments,SuperSegments);
458
459 FreeSegmentList(Segments,0);
460
461 FreeSegmentList(SuperSegments,0);
462
463 Segments=MergedSegments;
464
465 /* Re-index the merged segments */
466
467 IndexSegments(Segments,Nodes,Ways);
468
469 /* Cross reference the nodes and segments */
470
471 printf("\nCross-Reference Nodes and Segments\n==================================\n\n");
472 fflush(stdout);
473
474 /* Sort the nodes and segments geographically */
475
476 SortNodeListGeographically(Nodes);
477
478 SortSegmentListGeographically(Segments,Nodes);
479
480 /* Re-index the segments */
481
482 IndexSegments(Segments,Nodes,Ways);
483
484 /* Sort the turn relations geographically */
485
486 SortTurnRelationListGeographically(Relations,Nodes,Segments);
487
488 /* Output the results */
489
490 printf("\nWrite Out Database Files\n========================\n\n");
491 fflush(stdout);
492
493 /* Write out the nodes */
494
495 SaveNodeList(Nodes,FileName(dirname,prefix,"nodes.mem"),Segments);
496
497 FreeNodeList(Nodes,0);
498
499 /* Write out the segments */
500
501 SaveSegmentList(Segments,FileName(dirname,prefix,"segments.mem"));
502
503 FreeSegmentList(Segments,0);
504
505 /* Write out the ways */
506
507 SaveWayList(Ways,FileName(dirname,prefix,"ways.mem"));
508
509 FreeWayList(Ways,0);
510
511 /* Write out the relations */
512
513 SaveRelationList(Relations,FileName(dirname,prefix,"relations.mem"));
514
515 FreeRelationList(Relations,0);
516
517 /* Close the error log file */
518
519 if(errorlog)
520 close_errorlog();
521
522 /* Print the total time */
523
524 if(option_logtime)
525 {
526 printf("\n");
527 fprintf_elapsed_time(stdout,&start_time);
528 printf("Complete\n");
529 fflush(stdout);
530 }
531
532 return(0);
533 }
534
535
536 /*++++++++++++++++++++++++++++++++++++++
537 Print out the usage information.
538
539 int detail The level of detail to use - 0 = low, 1 = high.
540
541 const char *argerr The argument that gave the error (if there is one).
542
543 const char *err Other error message (if there is one).
544 ++++++++++++++++++++++++++++++++++++++*/
545
546 static void print_usage(int detail,const char *argerr,const char *err)
547 {
548 fprintf(stderr,
549 "Usage: planetsplitter [--help]\n"
550 " [--dir=<dirname>] [--prefix=<name>]\n"
551 #if defined(USE_PTHREADS) && USE_PTHREADS
552 " [--sort-ram-size=<size>] [--sort-threads=<number>]\n"
553 #else
554 " [--sort-ram-size=<size>]\n"
555 #endif
556 " [--tmpdir=<dirname>]\n"
557 " [--tagging=<filename>]\n"
558 " [--loggable] [--logtime]\n"
559 " [--errorlog[=<name>]]\n"
560 " [--parse-only | --process-only]\n"
561 " [--append] [--keep] [--changes]\n"
562 " [--max-iterations=<number>]\n"
563 " [--prune-none]\n"
564 " [--prune-isolated=<len>]\n"
565 " [--prune-short=<len>]\n"
566 " [--prune-straight=<len>]\n"
567 " [<filename.osm> ...]\n");
568
569 if(argerr)
570 fprintf(stderr,
571 "\n"
572 "Error with command line parameter: %s\n",argerr);
573
574 if(err)
575 fprintf(stderr,
576 "\n"
577 "Error: %s\n",err);
578
579 if(detail)
580 fprintf(stderr,
581 "\n"
582 "--help Prints this information.\n"
583 "\n"
584 "--dir=<dirname> The directory containing the routing database.\n"
585 "--prefix=<name> The filename prefix for the routing database.\n"
586 "\n"
587 "--sort-ram-size=<size> The amount of RAM (in MB) to use for data sorting\n"
588 #if SLIM
589 " (defaults to 64MB otherwise.)\n"
590 #else
591 " (defaults to 256MB otherwise.)\n"
592 #endif
593 #if defined(USE_PTHREADS) && USE_PTHREADS
594 "--sort-threads=<number> The number of threads to use for data sorting.\n"
595 #endif
596 "\n"
597 "--tmpdir=<dirname> The directory name for temporary files.\n"
598 " (defaults to the '--dir' option directory.)\n"
599 "\n"
600 "--tagging=<filename> The name of the XML file containing the tagging rules\n"
601 " (defaults to 'tagging.xml' with '--dir' and\n"
602 " '--prefix' options or the file installed in\n"
603 " '" DATADIR "').\n"
604 "\n"
605 "--loggable Print progress messages suitable for logging to file.\n"
606 "--logtime Print the elapsed time for each processing step.\n"
607 "--errorlog[=<name>] Log parsing errors to 'error.log' or the given name\n"
608 " (the '--dir' and '--prefix' options are applied).\n"
609 "\n"
610 "--parse-only Parse the OSM/OSC file(s) and store the results.\n"
611 "--process-only Process the stored results from previous option.\n"
612 "--append Parse the OSM file(s) and append to existing results.\n"
613 "--keep Keep the intermediate files after parsing & sorting.\n"
614 "--changes Parse the data as an OSC file and apply the changes.\n"
615 "\n"
616 "--max-iterations=<number> The number of iterations for finding super-nodes\n"
617 " (defaults to 5).\n"
618 "\n"
619 "--prune-none Disable the prune options below, they are re-enabled\n"
620 " by adding them to the command line after this option.\n"
621 "--prune-isolated=<len> Remove access from small disconnected segment groups\n"
622 " (defaults to removing groups under 500m).\n"
623 "--prune-short=<len> Remove short segments (defaults to removing segments\n"
624 " up to a maximum length of 5m).\n"
625 "--prune-straight=<len> Remove nodes in almost straight highways (defaults to\n"
626 " removing nodes up to 3m offset from a straight line).\n"
627 "\n"
628 "<filename.osm> ... The name(s) of the file(s) to process (by default\n"
629 " data is read from standard input).\n"
630 "\n"
631 "<transport> defaults to all but can be set to:\n"
632 "%s"
633 "\n"
634 "<highway> can be selected from:\n"
635 "%s"
636 "\n"
637 "<property> can be selected from:\n"
638 "%s",
639 TransportList(),HighwayList(),PropertyList());
640
641 exit(!detail);
642 }

Properties

Name Value
cvs:description Planet file splitter.
Index: This Site Routino Software Download Source Code Documentation XML Files Links
Index: View SVN Development Recent changes Development (destination-access branch) Version 3.4.1 (destination-access branch) Version 3.4.1 Version 3.4 (destination-access branch) Version 3.4 Version 3.3.3 (destination-access branch) Version 3.3.3 Version 3.3.2 (destination-access branch) Version 3.3.2 Version 3.3.1 (destination-access branch) Version 3.3.1 Version 3.3 (destination-access branch) Version 3.3 Version 3.2 Version 3.1.1 Version 3.1 Version 3.0 Version 2.7.3 Version 2.7.2 Version 2.7.1 Version 2.7 Version 2.6 Version 2.5.1 Version 2.5 Version 2.4.1 Version 2.4 Version 2.3.2 Version 2.3.1 Version 2.3 Version 2.2 Version 2.1.2 Version 2.1.1 Version 2.1 Version 2.0.3 Version 2.0.2 Version 2.0.1 Version 2.0 Version 1.5.1 Version 1.5 Version 1.4.1 Version 1.4 Version 1.3 Version 1.2 Version 1.1 Version 1.0
Powered by ViewVC 1.3.0-dev ViewVC Help
© Andrew M. Bishop = < amb @ ... >
Root