Routino SVN Repository Browser

Check out the latest version of Routino: svn co http://routino.org/svn/trunk routino

/[routino]/trunk/src/planetsplitter.c
ViewVC logotype

Contents of /trunk/src/planetsplitter.c

Parent Directory Parent Directory | Revision Log Revision Log

Revision 1186 - (show annotations) (download) (as text)
Mon Dec 10 18:36:08 2012 UTC (12 years, 3 months ago) by amb
File MIME type: text/x-csrc
File size: 18783 byte(s) 
New XML parser doesn't use stdio buffered file access but lower level read
functions.
1 /***************************************
2 OSM planet file splitter.
3
4 Part of the Routino routing software.
5 ******************/ /******************
6 This file Copyright 2008-2012 Andrew M. Bishop
7
8 This program is free software: you can redistribute it and/or modify
9 it under the terms of the GNU Affero General Public License as published by
10 the Free Software Foundation, either version 3 of the License, or
11 (at your option) any later version.
12
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU Affero General Public License for more details.
17
18 You should have received a copy of the GNU Affero General Public License
19 along with this program. If not, see <http://www.gnu.org/licenses/>.
20 ***************************************/
21
22
23 #include <stdio.h>
24 #include <stdlib.h>
25 #include <string.h>
26 #include <errno.h>
27 #include <sys/time.h>
28
29 #include "types.h"
30 #include "ways.h"
31
32 #include "typesx.h"
33 #include "nodesx.h"
34 #include "segmentsx.h"
35 #include "waysx.h"
36 #include "relationsx.h"
37 #include "superx.h"
38 #include "prunex.h"
39
40 #include "files.h"
41 #include "logging.h"
42 #include "functions.h"
43 #include "osmparser.h"
44 #include "tagging.h"
45
46
47 /* Global variables */
48
49 /*+ The name of the temporary directory. +*/
50 char *option_tmpdirname=NULL;
51
52 /*+ The amount of RAM to use for filesorting. +*/
53 size_t option_filesort_ramsize=0;
54
55 /*+ The number of threads to use for filesorting. +*/
56 int option_filesort_threads=1;
57
58
59 /* Local functions */
60
61 static void print_usage(int detail,const char *argerr,const char *err);
62
63
64 /*++++++++++++++++++++++++++++++++++++++
65 The main program for the planetsplitter.
66 ++++++++++++++++++++++++++++++++++++++*/
67
68 int main(int argc,char** argv)
69 {
70 struct timeval start_time;
71 NodesX *Nodes;
72 SegmentsX *Segments,*SuperSegments=NULL,*MergedSegments=NULL;
73 WaysX *Ways;
74 RelationsX *Relations;
75 int iteration=0,quit=0;
76 int max_iterations=5;
77 char *dirname=NULL,*prefix=NULL,*tagging=NULL,*errorlog=NULL;
78 int option_parse_only=0,option_process_only=0;
79 int option_append=0,option_keep=0,option_changes=0;
80 int option_filenames=0;
81 int option_prune_isolated=500,option_prune_short=5,option_prune_straight=3;
82 int arg;
83
84 gettimeofday(&start_time,NULL);
85
86 /* Parse the command line arguments */
87
88 for(arg=1;arg<argc;arg++)
89 {
90 if(!strcmp(argv[arg],"--help"))
91 print_usage(1,NULL,NULL);
92 else if(!strncmp(argv[arg],"--sort-ram-size=",16))
93 option_filesort_ramsize=atoi(&argv[arg][16]);
94 #if defined(USE_PTHREADS) && USE_PTHREADS
95 else if(!strncmp(argv[arg],"--sort-threads=",15))
96 option_filesort_threads=atoi(&argv[arg][15]);
97 #endif
98 else if(!strncmp(argv[arg],"--dir=",6))
99 dirname=&argv[arg][6];
100 else if(!strncmp(argv[arg],"--tmpdir=",9))
101 option_tmpdirname=&argv[arg][9];
102 else if(!strncmp(argv[arg],"--prefix=",9))
103 prefix=&argv[arg][9];
104 else if(!strcmp(argv[arg],"--parse-only"))
105 option_parse_only=1;
106 else if(!strcmp(argv[arg],"--process-only"))
107 option_process_only=1;
108 else if(!strcmp(argv[arg],"--append"))
109 option_append=1;
110 else if(!strcmp(argv[arg],"--keep"))
111 option_keep=1;
112 else if(!strcmp(argv[arg],"--changes"))
113 option_changes=1;
114 else if(!strcmp(argv[arg],"--loggable"))
115 option_loggable=1;
116 else if(!strcmp(argv[arg],"--logtime"))
117 option_logtime=1;
118 else if(!strcmp(argv[arg],"--errorlog"))
119 errorlog="error.log";
120 else if(!strncmp(argv[arg],"--errorlog=",11))
121 errorlog=&argv[arg][11];
122 else if(!strncmp(argv[arg],"--max-iterations=",17))
123 max_iterations=atoi(&argv[arg][17]);
124 else if(!strncmp(argv[arg],"--tagging=",10))
125 tagging=&argv[arg][10];
126 else if(!strncmp(argv[arg],"--prune",7))
127 {
128 if(!strcmp(&argv[arg][7],"-none"))
129 option_prune_isolated=option_prune_short=option_prune_straight=0;
130 else if(!strncmp(&argv[arg][7],"-isolated=",10))
131 option_prune_isolated=atoi(&argv[arg][17]);
132 else if(!strncmp(&argv[arg][7],"-short=",7))
133 option_prune_short=atoi(&argv[arg][14]);
134 else if(!strncmp(&argv[arg][7],"-straight=",10))
135 option_prune_straight=atoi(&argv[arg][17]);
136 else
137 print_usage(0,argv[arg],NULL);
138 }
139 else if(argv[arg][0]=='-' && argv[arg][1]=='-')
140 print_usage(0,argv[arg],NULL);
141 else
142 option_filenames++;
143 }
144
145 /* Check the specified command line options */
146
147 if(option_parse_only && option_process_only)
148 print_usage(0,NULL,"Cannot use '--parse-only' and '--process-only' at the same time.");
149
150 if(option_append && option_process_only)
151 print_usage(0,NULL,"Cannot use '--append' and '--process-only' at the same time.");
152
153 if(option_filenames && option_process_only)
154 print_usage(0,NULL,"Cannot use '--process-only' and filenames at the same time.");
155
156 if(!option_filesort_ramsize)
157 {
158 #if SLIM
159 option_filesort_ramsize=64*1024*1024;
160 #else
161 option_filesort_ramsize=256*1024*1024;
162 #endif
163 }
164 else
165 option_filesort_ramsize*=1024*1024;
166
167 if(!option_tmpdirname)
168 {
169 if(!dirname)
170 option_tmpdirname=".";
171 else
172 option_tmpdirname=dirname;
173 }
174
175 if(!option_process_only)
176 {
177 if(tagging)
178 {
179 if(!ExistsFile(tagging))
180 {
181 fprintf(stderr,"Error: The '--tagging' option specifies a file that does not exist.\n");
182 return(1);
183 }
184 }
185 else
186 {
187 if(ExistsFile(FileName(dirname,prefix,"tagging.xml")))
188 tagging=FileName(dirname,prefix,"tagging.xml");
189 else if(ExistsFile(FileName(DATADIR,NULL,"tagging.xml")))
190 tagging=FileName(DATADIR,NULL,"tagging.xml");
191 else
192 {
193 fprintf(stderr,"Error: The '--tagging' option was not used and the default 'tagging.xml' does not exist.\n");
194 return(1);
195 }
196 }
197
198 if(ParseXMLTaggingRules(tagging))
199 {
200 fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging);
201 return(1);
202 }
203 }
204
205 /* Create new node, segment, way and relation variables */
206
207 Nodes=NewNodeList(option_append||option_changes,option_process_only);
208
209 Segments=NewSegmentList(option_append||option_changes,option_process_only);
210
211 Ways=NewWayList(option_append||option_changes,option_process_only);
212
213 Relations=NewRelationList(option_append||option_changes,option_process_only);
214
215 /* Create the error log file */
216
217 if(errorlog)
218 open_errorlog(FileName(dirname,prefix,errorlog),option_append||option_changes||option_process_only);
219
220 /* Parse the file */
221
222 if(!option_process_only)
223 {
224 if(option_filenames)
225 {
226 for(arg=1;arg<argc;arg++)
227 {
228 int fd;
229
230 if(argv[arg][0]=='-' && argv[arg][1]=='-')
231 continue;
232
233 fd=ReOpenFile(argv[arg]);
234
235 if(option_changes)
236 {
237 printf("\nParse OSC Data [%s]\n==============\n\n",argv[arg]);
238 fflush(stdout);
239
240 if(ParseOSC(fd,Nodes,Segments,Ways,Relations))
241 exit(EXIT_FAILURE);
242 }
243 else
244 {
245 printf("\nParse OSM Data [%s]\n==============\n\n",argv[arg]);
246 fflush(stdout);
247
248 if(ParseOSM(fd,Nodes,Segments,Ways,Relations))
249 exit(EXIT_FAILURE);
250 }
251
252 CloseFile(fd);
253 }
254 }
255 else
256 {
257 if(option_changes)
258 {
259 printf("\nParse OSC Data\n==============\n\n");
260 fflush(stdout);
261
262 if(ParseOSC(0,Nodes,Segments,Ways,Relations))
263 exit(EXIT_FAILURE);
264 }
265 else
266 {
267 printf("\nParse OSM Data\n==============\n\n");
268 fflush(stdout);
269
270 if(ParseOSM(0,Nodes,Segments,Ways,Relations))
271 exit(EXIT_FAILURE);
272 }
273 }
274
275 DeleteXMLTaggingRules();
276 }
277
278 FinishNodeList(Nodes);
279 FinishSegmentList(Segments);
280 FinishWayList(Ways);
281 FinishRelationList(Relations);
282
283 if(option_parse_only)
284 {
285 FreeNodeList(Nodes,1);
286 FreeSegmentList(Segments,1);
287 FreeWayList(Ways,1);
288 FreeRelationList(Relations,1);
289
290 return(0);
291 }
292
293
294 /* Sort the data */
295
296 printf("\nSort OSM Data\n=============\n\n");
297 fflush(stdout);
298
299 /* Sort the nodes, segments, ways and relations */
300
301 SortNodeList(Nodes);
302
303 if(option_changes)
304 ApplySegmentChanges(Segments);
305
306 SortSegmentList(Segments);
307
308 SortWayList(Ways);
309
310 SortRelationList(Relations);
311
312 /* Process the data */
313
314 printf("\nProcess OSM Data\n================\n\n");
315 fflush(stdout);
316
317 /* Extract the way names (must be before using the ways) */
318
319 ExtractWayNames(Ways,option_keep||option_changes);
320
321 /* Remove bad segments (must be after sorting the nodes, segments and ways) */
322
323 RemoveBadSegments(Segments,Nodes,Ways,option_keep||option_changes);
324
325 /* Remove non-highway nodes (must be after removing the bad segments) */
326
327 RemoveNonHighwayNodes(Nodes,Segments,option_keep||option_changes);
328
329 /* Process the route relations and first part of turn relations (must be before compacting the ways) */
330
331 ProcessRouteRelations(Relations,Ways,option_keep||option_changes);
332
333 ProcessTurnRelations1(Relations,Nodes,Ways,option_keep||option_changes);
334
335 /* Measure the segments and replace node/way id with index (must be after removing non-highway nodes) */
336
337 MeasureSegments(Segments,Nodes,Ways);
338
339 /* Index the segments */
340
341 IndexSegments(Segments,Nodes,Ways);
342
343 /* Convert the turn relations from ways into nodes */
344
345 ProcessTurnRelations2(Relations,Nodes,Segments,Ways);
346
347 /* Compact the ways (must be after turn relations 2) */
348
349 CompactWayList(Ways,Segments);
350
351 /* Index the segments */
352
353 IndexSegments(Segments,Nodes,Ways);
354
355 /* Prune unwanted nodes/segments. */
356
357 if(option_prune_straight || option_prune_isolated || option_prune_short)
358 {
359 printf("\nPrune Unneeded Data\n===================\n\n");
360 fflush(stdout);
361
362 StartPruning(Nodes,Segments,Ways);
363
364 if(option_prune_straight)
365 PruneStraightHighwayNodes(Nodes,Segments,Ways,option_prune_straight);
366
367 if(option_prune_isolated)
368 PruneIsolatedRegions(Nodes,Segments,Ways,option_prune_isolated);
369
370 if(option_prune_short)
371 PruneShortSegments(Nodes,Segments,Ways,option_prune_short);
372
373 FinishPruning(Nodes,Segments,Ways);
374
375 /* Remove the pruned nodes and segments and update the indexes */
376
377 RemovePrunedNodes(Nodes,Segments);
378 RemovePrunedSegments(Segments,Ways);
379 CompactWayList(Ways,Segments);
380 RemovePrunedTurnRelations(Relations,Nodes);
381 IndexSegments(Segments,Nodes,Ways);
382 }
383
384 /* Repeated iteration on Super-Nodes and Super-Segments */
385
386 do
387 {
388 int nsuper;
389
390 printf("\nProcess Super-Data (iteration %d)\n================================%s\n\n",iteration,iteration>9?"=":"");
391 fflush(stdout);
392
393 if(iteration==0)
394 {
395 /* Select the super-nodes */
396
397 ChooseSuperNodes(Nodes,Segments,Ways);
398
399 /* Select the super-segments */
400
401 SuperSegments=CreateSuperSegments(Nodes,Segments,Ways);
402
403 nsuper=Segments->number;
404 }
405 else
406 {
407 SegmentsX *SuperSegments2;
408
409 /* Select the super-nodes */
410
411 ChooseSuperNodes(Nodes,SuperSegments,Ways);
412
413 /* Select the super-segments */
414
415 SuperSegments2=CreateSuperSegments(Nodes,SuperSegments,Ways);
416
417 nsuper=SuperSegments->number;
418
419 FreeSegmentList(SuperSegments,0);
420
421 SuperSegments=SuperSegments2;
422 }
423
424 /* Sort the super-segments and remove duplicates */
425
426 DeduplicateSuperSegments(SuperSegments,Ways);
427
428 /* Index the segments */
429
430 IndexSegments(SuperSegments,Nodes,Ways);
431
432 /* Check for end condition */
433
434 if(SuperSegments->number==nsuper)
435 quit=1;
436
437 iteration++;
438
439 if(iteration>max_iterations)
440 quit=1;
441 }
442 while(!quit);
443
444 /* Combine the super-segments */
445
446 printf("\nCombine Segments and Super-Segments\n===================================\n\n");
447 fflush(stdout);
448
449 /* Merge the super-segments */
450
451 MergedSegments=MergeSuperSegments(Segments,SuperSegments);
452
453 FreeSegmentList(Segments,0);
454
455 FreeSegmentList(SuperSegments,0);
456
457 Segments=MergedSegments;
458
459 /* Re-index the merged segments */
460
461 IndexSegments(Segments,Nodes,Ways);
462
463 /* Cross reference the nodes and segments */
464
465 printf("\nCross-Reference Nodes and Segments\n==================================\n\n");
466 fflush(stdout);
467
468 /* Sort the nodes and segments geographically */
469
470 SortNodeListGeographically(Nodes);
471
472 SortSegmentListGeographically(Segments,Nodes);
473
474 /* Re-index the segments */
475
476 IndexSegments(Segments,Nodes,Ways);
477
478 /* Sort the turn relations geographically */
479
480 SortTurnRelationListGeographically(Relations,Nodes,Segments);
481
482 /* Output the results */
483
484 printf("\nWrite Out Database Files\n========================\n\n");
485 fflush(stdout);
486
487 /* Write out the nodes */
488
489 SaveNodeList(Nodes,FileName(dirname,prefix,"nodes.mem"),Segments);
490
491 FreeNodeList(Nodes,0);
492
493 /* Write out the segments */
494
495 SaveSegmentList(Segments,FileName(dirname,prefix,"segments.mem"));
496
497 FreeSegmentList(Segments,0);
498
499 /* Write out the ways */
500
501 SaveWayList(Ways,FileName(dirname,prefix,"ways.mem"));
502
503 FreeWayList(Ways,0);
504
505 /* Write out the relations */
506
507 SaveRelationList(Relations,FileName(dirname,prefix,"relations.mem"));
508
509 FreeRelationList(Relations,0);
510
511 /* Close the error log file */
512
513 if(errorlog)
514 close_errorlog();
515
516 /* Print the total time */
517
518 if(option_logtime)
519 {
520 printf("\n");
521 fprintf_elapsed_time(stdout,&start_time);
522 printf("Complete\n");
523 fflush(stdout);
524 }
525
526 return(0);
527 }
528
529
530 /*++++++++++++++++++++++++++++++++++++++
531 Print out the usage information.
532
533 int detail The level of detail to use - 0 = low, 1 = high.
534
535 const char *argerr The argument that gave the error (if there is one).
536
537 const char *err Other error message (if there is one).
538 ++++++++++++++++++++++++++++++++++++++*/
539
540 static void print_usage(int detail,const char *argerr,const char *err)
541 {
542 fprintf(stderr,
543 "Usage: planetsplitter [--help]\n"
544 " [--dir=<dirname>] [--prefix=<name>]\n"
545 #if defined(USE_PTHREADS) && USE_PTHREADS
546 " [--sort-ram-size=<size>] [--sort-threads=<number>]\n"
547 #else
548 " [--sort-ram-size=<size>]\n"
549 #endif
550 " [--tmpdir=<dirname>]\n"
551 " [--tagging=<filename>]\n"
552 " [--loggable] [--logtime]\n"
553 " [--errorlog[=<name>]]\n"
554 " [--parse-only | --process-only]\n"
555 " [--append] [--keep] [--changes]\n"
556 " [--max-iterations=<number>]\n"
557 " [--prune-none]\n"
558 " [--prune-isolated=<len>]\n"
559 " [--prune-short=<len>]\n"
560 " [--prune-straight=<len>]\n"
561 " [<filename.osm> ...]\n");
562
563 if(argerr)
564 fprintf(stderr,
565 "\n"
566 "Error with command line parameter: %s\n",argerr);
567
568 if(err)
569 fprintf(stderr,
570 "\n"
571 "Error: %s\n",err);
572
573 if(detail)
574 fprintf(stderr,
575 "\n"
576 "--help Prints this information.\n"
577 "\n"
578 "--dir=<dirname> The directory containing the routing database.\n"
579 "--prefix=<name> The filename prefix for the routing database.\n"
580 "\n"
581 "--sort-ram-size=<size> The amount of RAM (in MB) to use for data sorting\n"
582 #if SLIM
583 " (defaults to 64MB otherwise.)\n"
584 #else
585 " (defaults to 256MB otherwise.)\n"
586 #endif
587 #if defined(USE_PTHREADS) && USE_PTHREADS
588 "--sort-threads=<number> The number of threads to use for data sorting.\n"
589 #endif
590 "\n"
591 "--tmpdir=<dirname> The directory name for temporary files.\n"
592 " (defaults to the '--dir' option directory.)\n"
593 "\n"
594 "--tagging=<filename> The name of the XML file containing the tagging rules\n"
595 " (defaults to 'tagging.xml' with '--dir' and\n"
596 " '--prefix' options or the file installed in\n"
597 " '" DATADIR "').\n"
598 "\n"
599 "--loggable Print progress messages suitable for logging to file.\n"
600 "--logtime Print the elapsed time for each processing step.\n"
601 "--errorlog[=<name>] Log parsing errors to 'error.log' or the given name\n"
602 " (the '--dir' and '--prefix' options are applied).\n"
603 "\n"
604 "--parse-only Parse the OSM/OSC file(s) and store the results.\n"
605 "--process-only Process the stored results from previous option.\n"
606 "--append Parse the OSM file(s) and append to existing results.\n"
607 "--keep Keep the intermediate files after parsing & sorting.\n"
608 "--changes Parse the data as an OSC file and apply the changes.\n"
609 "\n"
610 "--max-iterations=<number> The number of iterations for finding super-nodes\n"
611 " (defaults to 5).\n"
612 "\n"
613 "--prune-none Disable the prune options below, they are re-enabled\n"
614 " by adding them to the command line after this option.\n"
615 "--prune-isolated=<len> Remove access from small disconnected segment groups\n"
616 " (defaults to removing groups under 500m).\n"
617 "--prune-short=<len> Remove short segments (defaults to removing segments\n"
618 " up to a maximum length of 5m).\n"
619 "--prune-straight=<len> Remove nodes in almost straight highways (defaults to\n"
620 " removing nodes up to 3m offset from a straight line).\n"
621 "\n"
622 "<filename.osm> ... The name(s) of the file(s) to process (by default\n"
623 " data is read from standard input).\n"
624 "\n"
625 "<transport> defaults to all but can be set to:\n"
626 "%s"
627 "\n"
628 "<highway> can be selected from:\n"
629 "%s"
630 "\n"
631 "<property> can be selected from:\n"
632 "%s",
633 TransportList(),HighwayList(),PropertyList());
634
635 exit(!detail);
636 }

Properties

Name Value
cvs:description Planet file splitter.
Index: This Site Routino Software Download Source Code Documentation XML Files Links
Index: View SVN Development Recent changes Development (destination-access branch) Version 3.4.1 (destination-access branch) Version 3.4.1 Version 3.4 (destination-access branch) Version 3.4 Version 3.3.3 (destination-access branch) Version 3.3.3 Version 3.3.2 (destination-access branch) Version 3.3.2 Version 3.3.1 (destination-access branch) Version 3.3.1 Version 3.3 (destination-access branch) Version 3.3 Version 3.2 Version 3.1.1 Version 3.1 Version 3.0 Version 2.7.3 Version 2.7.2 Version 2.7.1 Version 2.7 Version 2.6 Version 2.5.1 Version 2.5 Version 2.4.1 Version 2.4 Version 2.3.2 Version 2.3.1 Version 2.3 Version 2.2 Version 2.1.2 Version 2.1.1 Version 2.1 Version 2.0.3 Version 2.0.2 Version 2.0.1 Version 2.0 Version 1.5.1 Version 1.5 Version 1.4.1 Version 1.4 Version 1.3 Version 1.2 Version 1.1 Version 1.0
Powered by ViewVC 1.3.0-dev ViewVC Help
© Andrew M. Bishop = < amb @ ... >
Root