Routino SVN Repository Browser

Check out the latest version of Routino: svn co http://routino.org/svn/trunk routino

/[routino]/trunk/src/planetsplitter.c
ViewVC logotype

Contents of /trunk/src/planetsplitter.c

Parent Directory Parent Directory | Revision Log Revision Log

Revision 1136 - (show annotations) (download) (as text)
Sat Nov 10 19:23:32 2012 UTC (12 years, 4 months ago) by amb
File MIME type: text/x-csrc
File size: 17885 byte(s) 
Added a --preserve option which keeps the raw data files after parsing, sorting
and de-duplication.
1 /***************************************
2 OSM planet file splitter.
3
4 Part of the Routino routing software.
5 ******************/ /******************
6 This file Copyright 2008-2012 Andrew M. Bishop
7
8 This program is free software: you can redistribute it and/or modify
9 it under the terms of the GNU Affero General Public License as published by
10 the Free Software Foundation, either version 3 of the License, or
11 (at your option) any later version.
12
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU Affero General Public License for more details.
17
18 You should have received a copy of the GNU Affero General Public License
19 along with this program. If not, see <http://www.gnu.org/licenses/>.
20 ***************************************/
21
22
23 #include <stdio.h>
24 #include <stdlib.h>
25 #include <string.h>
26 #include <errno.h>
27 #include <sys/time.h>
28
29 #include "types.h"
30 #include "ways.h"
31
32 #include "typesx.h"
33 #include "nodesx.h"
34 #include "segmentsx.h"
35 #include "waysx.h"
36 #include "relationsx.h"
37 #include "superx.h"
38 #include "prunex.h"
39
40 #include "files.h"
41 #include "logging.h"
42 #include "functions.h"
43 #include "osmparser.h"
44 #include "tagging.h"
45
46
47 /* Global variables */
48
49 /*+ The name of the temporary directory. +*/
50 char *option_tmpdirname=NULL;
51
52 /*+ The amount of RAM to use for filesorting. +*/
53 size_t option_filesort_ramsize=0;
54
55 /*+ The number of threads to use for filesorting. +*/
56 int option_filesort_threads=1;
57
58
59 /* Local functions */
60
61 static void print_usage(int detail,const char *argerr,const char *err);
62
63
64 /*++++++++++++++++++++++++++++++++++++++
65 The main program for the planetsplitter.
66 ++++++++++++++++++++++++++++++++++++++*/
67
68 int main(int argc,char** argv)
69 {
70 struct timeval start_time;
71 NodesX *Nodes;
72 SegmentsX *Segments,*SuperSegments=NULL,*MergedSegments=NULL;
73 WaysX *Ways;
74 RelationsX *Relations;
75 int iteration=0,quit=0;
76 int max_iterations=5;
77 char *dirname=NULL,*prefix=NULL,*tagging=NULL,*errorlog=NULL;
78 int option_parse_only=0,option_process_only=0,option_append=0,option_preserve=0;
79 int option_filenames=0;
80 int option_prune_isolated=500,option_prune_short=5,option_prune_straight=3;
81 int arg;
82
83 gettimeofday(&start_time,NULL);
84
85 /* Parse the command line arguments */
86
87 for(arg=1;arg<argc;arg++)
88 {
89 if(!strcmp(argv[arg],"--help"))
90 print_usage(1,NULL,NULL);
91 else if(!strncmp(argv[arg],"--sort-ram-size=",16))
92 option_filesort_ramsize=atoi(&argv[arg][16]);
93 #if defined(USE_PTHREADS) && USE_PTHREADS
94 else if(!strncmp(argv[arg],"--sort-threads=",15))
95 option_filesort_threads=atoi(&argv[arg][15]);
96 #endif
97 else if(!strncmp(argv[arg],"--dir=",6))
98 dirname=&argv[arg][6];
99 else if(!strncmp(argv[arg],"--tmpdir=",9))
100 option_tmpdirname=&argv[arg][9];
101 else if(!strncmp(argv[arg],"--prefix=",9))
102 prefix=&argv[arg][9];
103 else if(!strcmp(argv[arg],"--parse-only"))
104 option_parse_only=1;
105 else if(!strcmp(argv[arg],"--process-only"))
106 option_process_only=1;
107 else if(!strcmp(argv[arg],"--append"))
108 option_append=1;
109 else if(!strcmp(argv[arg],"--preserve"))
110 option_preserve=1;
111 else if(!strcmp(argv[arg],"--loggable"))
112 option_loggable=1;
113 else if(!strcmp(argv[arg],"--logtime"))
114 option_logtime=1;
115 else if(!strcmp(argv[arg],"--errorlog"))
116 errorlog="error.log";
117 else if(!strncmp(argv[arg],"--errorlog=",11))
118 errorlog=&argv[arg][11];
119 else if(!strncmp(argv[arg],"--max-iterations=",17))
120 max_iterations=atoi(&argv[arg][17]);
121 else if(!strncmp(argv[arg],"--tagging=",10))
122 tagging=&argv[arg][10];
123 else if(!strncmp(argv[arg],"--prune",7))
124 {
125 if(!strcmp(&argv[arg][7],"-none"))
126 option_prune_isolated=option_prune_short=option_prune_straight=0;
127 else if(!strncmp(&argv[arg][7],"-isolated=",10))
128 option_prune_isolated=atoi(&argv[arg][17]);
129 else if(!strncmp(&argv[arg][7],"-short=",7))
130 option_prune_short=atoi(&argv[arg][14]);
131 else if(!strncmp(&argv[arg][7],"-straight=",10))
132 option_prune_straight=atoi(&argv[arg][17]);
133 else
134 print_usage(0,argv[arg],NULL);
135 }
136 else if(argv[arg][0]=='-' && argv[arg][1]=='-')
137 print_usage(0,argv[arg],NULL);
138 else
139 option_filenames++;
140 }
141
142 /* Check the specified command line options */
143
144 if(option_parse_only && option_process_only)
145 print_usage(0,NULL,"Cannot use '--parse-only' and '--process-only' at the same time.");
146
147 if(option_append && option_process_only)
148 print_usage(0,NULL,"Cannot use '--append' and '--process-only' at the same time.");
149
150 if(option_filenames && option_process_only)
151 print_usage(0,NULL,"Cannot use '--process-only' and filenames at the same time.");
152
153 if(!option_filesort_ramsize)
154 {
155 #if SLIM
156 option_filesort_ramsize=64*1024*1024;
157 #else
158 option_filesort_ramsize=256*1024*1024;
159 #endif
160 }
161 else
162 option_filesort_ramsize*=1024*1024;
163
164 if(!option_tmpdirname)
165 {
166 if(!dirname)
167 option_tmpdirname=".";
168 else
169 option_tmpdirname=dirname;
170 }
171
172 if(!option_process_only)
173 {
174 if(tagging)
175 {
176 if(!ExistsFile(tagging))
177 {
178 fprintf(stderr,"Error: The '--tagging' option specifies a file that does not exist.\n");
179 return(1);
180 }
181 }
182 else
183 {
184 if(ExistsFile(FileName(dirname,prefix,"tagging.xml")))
185 tagging=FileName(dirname,prefix,"tagging.xml");
186 else if(ExistsFile(FileName(DATADIR,NULL,"tagging.xml")))
187 tagging=FileName(DATADIR,NULL,"tagging.xml");
188 else
189 {
190 fprintf(stderr,"Error: The '--tagging' option was not used and the default 'tagging.xml' does not exist.\n");
191 return(1);
192 }
193 }
194
195 if(ParseXMLTaggingRules(tagging))
196 {
197 fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging);
198 return(1);
199 }
200 }
201
202 /* Create new node, segment, way and relation variables */
203
204 Nodes=NewNodeList(option_append,option_process_only);
205
206 Segments=NewSegmentList(option_append,option_process_only);
207
208 Ways=NewWayList(option_append,option_process_only);
209
210 Relations=NewRelationList(option_append,option_process_only);
211
212 /* Create the error log file */
213
214 if(errorlog)
215 open_errorlog(FileName(dirname,prefix,errorlog),option_append||option_process_only);
216
217 /* Parse the file */
218
219 if(!option_process_only)
220 {
221 if(option_filenames)
222 {
223 for(arg=1;arg<argc;arg++)
224 {
225 FILE *file;
226
227 if(argv[arg][0]=='-' && argv[arg][1]=='-')
228 continue;
229
230 file=fopen(argv[arg],"rb");
231
232 if(!file)
233 {
234 fprintf(stderr,"Cannot open file '%s' for reading [%s].\n",argv[arg],strerror(errno));
235 exit(EXIT_FAILURE);
236 }
237
238 printf("\nParse OSM Data [%s]\n==============\n\n",argv[arg]);
239 fflush(stdout);
240
241 if(ParseOSM(file,Nodes,Segments,Ways,Relations))
242 exit(EXIT_FAILURE);
243
244 fclose(file);
245 }
246 }
247 else
248 {
249 printf("\nParse OSM Data\n==============\n\n");
250 fflush(stdout);
251
252 if(ParseOSM(stdin,Nodes,Segments,Ways,Relations))
253 exit(EXIT_FAILURE);
254 }
255
256 DeleteXMLTaggingRules();
257 }
258
259 if(option_parse_only)
260 {
261 FreeNodeList(Nodes);
262 FreeSegmentList(Segments);
263 FreeWayList(Ways);
264 FreeRelationList(Relations);
265
266 return(0);
267 }
268
269 FinishNodeList(Nodes);
270 FinishSegmentList(Segments);
271 FinishWayList(Ways);
272 FinishRelationList(Relations);
273
274
275 /* Process the data */
276
277 printf("\nProcess OSM Data\n================\n\n");
278 fflush(stdout);
279
280 /* Sort the nodes, segments, ways and relations */
281
282 SortNodeList(Nodes);
283
284 SortSegmentList(Segments);
285
286 SortWayList(Ways);
287
288 SortRelationList(Relations);
289
290 /* Remove bad segments (must be after sorting the nodes and segments) */
291
292 RemoveBadSegments(Segments,Nodes,option_preserve);
293
294 /* Remove non-highway nodes (must be after removing the bad segments) */
295
296 RemoveNonHighwayNodes(Nodes,Segments,option_preserve);
297
298 /* Extract the way names (must be before using the ways) */
299
300 ExtractWayNames(Ways,option_preserve);
301
302 /* Process the route relations and first part of turn relations (must be before compacting the ways) */
303
304 ProcessRouteRelations(Relations,Ways,option_preserve);
305
306 ProcessTurnRelations1(Relations,Nodes,Ways,option_preserve);
307
308 /* Measure the segments and replace node/way id with index (must be after removing non-highway nodes) */
309
310 MeasureSegments(Segments,Nodes,Ways);
311
312 /* Index the segments */
313
314 IndexSegments(Segments,Nodes,Ways);
315
316 /* Convert the turn relations from ways into nodes */
317
318 ProcessTurnRelations2(Relations,Nodes,Segments,Ways);
319
320 /* Compact the ways (must be after turn relations 2) */
321
322 CompactWayList(Ways,Segments);
323
324 /* Index the segments */
325
326 IndexSegments(Segments,Nodes,Ways);
327
328 /* Prune unwanted nodes/segments. */
329
330 if(option_prune_straight || option_prune_isolated || option_prune_short)
331 {
332 printf("\nPrune Unneeded Data\n===================\n\n");
333 fflush(stdout);
334
335 StartPruning(Nodes,Segments,Ways);
336
337 if(option_prune_straight)
338 PruneStraightHighwayNodes(Nodes,Segments,Ways,option_prune_straight);
339
340 if(option_prune_isolated)
341 PruneIsolatedRegions(Nodes,Segments,Ways,option_prune_isolated);
342
343 if(option_prune_short)
344 PruneShortSegments(Nodes,Segments,Ways,option_prune_short);
345
346 FinishPruning(Nodes,Segments,Ways);
347
348 /* Remove the pruned nodes and segments and update the indexes */
349
350 RemovePrunedNodes(Nodes,Segments);
351 RemovePrunedSegments(Segments,Ways);
352 CompactWayList(Ways,Segments);
353 RemovePrunedTurnRelations(Relations,Nodes);
354 IndexSegments(Segments,Nodes,Ways);
355 }
356
357 /* Repeated iteration on Super-Nodes and Super-Segments */
358
359 do
360 {
361 int nsuper;
362
363 printf("\nProcess Super-Data (iteration %d)\n================================%s\n\n",iteration,iteration>9?"=":"");
364 fflush(stdout);
365
366 if(iteration==0)
367 {
368 /* Select the super-nodes */
369
370 ChooseSuperNodes(Nodes,Segments,Ways);
371
372 /* Select the super-segments */
373
374 SuperSegments=CreateSuperSegments(Nodes,Segments,Ways);
375
376 nsuper=Segments->number;
377 }
378 else
379 {
380 SegmentsX *SuperSegments2;
381
382 /* Select the super-nodes */
383
384 ChooseSuperNodes(Nodes,SuperSegments,Ways);
385
386 /* Select the super-segments */
387
388 SuperSegments2=CreateSuperSegments(Nodes,SuperSegments,Ways);
389
390 nsuper=SuperSegments->number;
391
392 FreeSegmentList(SuperSegments);
393
394 SuperSegments=SuperSegments2;
395 }
396
397 /* Sort the super-segments and remove duplicates */
398
399 DeduplicateSuperSegments(SuperSegments,Ways);
400
401 /* Index the segments */
402
403 IndexSegments(SuperSegments,Nodes,Ways);
404
405 /* Check for end condition */
406
407 if(SuperSegments->number==nsuper)
408 quit=1;
409
410 iteration++;
411
412 if(iteration>max_iterations)
413 quit=1;
414 }
415 while(!quit);
416
417 /* Combine the super-segments */
418
419 printf("\nCombine Segments and Super-Segments\n===================================\n\n");
420 fflush(stdout);
421
422 /* Merge the super-segments */
423
424 MergedSegments=MergeSuperSegments(Segments,SuperSegments);
425
426 FreeSegmentList(Segments);
427
428 FreeSegmentList(SuperSegments);
429
430 Segments=MergedSegments;
431
432 /* Re-index the merged segments */
433
434 IndexSegments(Segments,Nodes,Ways);
435
436 /* Cross reference the nodes and segments */
437
438 printf("\nCross-Reference Nodes and Segments\n==================================\n\n");
439 fflush(stdout);
440
441 /* Sort the nodes and segments geographically */
442
443 SortNodeListGeographically(Nodes);
444
445 SortSegmentListGeographically(Segments,Nodes);
446
447 /* Re-index the segments */
448
449 IndexSegments(Segments,Nodes,Ways);
450
451 /* Sort the turn relations geographically */
452
453 SortTurnRelationListGeographically(Relations,Nodes,Segments);
454
455 /* Output the results */
456
457 printf("\nWrite Out Database Files\n========================\n\n");
458 fflush(stdout);
459
460 /* Write out the nodes */
461
462 SaveNodeList(Nodes,FileName(dirname,prefix,"nodes.mem"),Segments);
463
464 FreeNodeList(Nodes);
465
466 /* Write out the segments */
467
468 SaveSegmentList(Segments,FileName(dirname,prefix,"segments.mem"));
469
470 FreeSegmentList(Segments);
471
472 /* Write out the ways */
473
474 SaveWayList(Ways,FileName(dirname,prefix,"ways.mem"));
475
476 FreeWayList(Ways);
477
478 /* Write out the relations */
479
480 SaveRelationList(Relations,FileName(dirname,prefix,"relations.mem"));
481
482 FreeRelationList(Relations);
483
484 /* Close the error log file */
485
486 if(errorlog)
487 close_errorlog();
488
489 /* Print the total time */
490
491 if(option_logtime)
492 {
493 printf("\n");
494 fprintf_elapsed_time(stdout,&start_time);
495 printf("Complete\n");
496 fflush(stdout);
497 }
498
499 return(0);
500 }
501
502
503 /*++++++++++++++++++++++++++++++++++++++
504 Print out the usage information.
505
506 int detail The level of detail to use - 0 = low, 1 = high.
507
508 const char *argerr The argument that gave the error (if there is one).
509
510 const char *err Other error message (if there is one).
511 ++++++++++++++++++++++++++++++++++++++*/
512
513 static void print_usage(int detail,const char *argerr,const char *err)
514 {
515 fprintf(stderr,
516 "Usage: planetsplitter [--help]\n"
517 " [--dir=<dirname>] [--prefix=<name>]\n"
518 #if defined(USE_PTHREADS) && USE_PTHREADS
519 " [--sort-ram-size=<size>] [--sort-threads=<number>]\n"
520 #else
521 " [--sort-ram-size=<size>]\n"
522 #endif
523 " [--tmpdir=<dirname>]\n"
524 " [--tagging=<filename>]\n"
525 " [--loggable] [--logtime]\n"
526 " [--errorlog[=<name>]]\n"
527 " [--parse-only | --process-only]\n"
528 " [--append]\n"
529 " [--max-iterations=<number>]\n"
530 " [--prune-none]\n"
531 " [--prune-isolated=<len>]\n"
532 " [--prune-short=<len>]\n"
533 " [--prune-straight=<len>]\n"
534 " [<filename.osm> ...]\n");
535
536 if(argerr)
537 fprintf(stderr,
538 "\n"
539 "Error with command line parameter: %s\n",argerr);
540
541 if(err)
542 fprintf(stderr,
543 "\n"
544 "Error: %s\n",err);
545
546 if(detail)
547 fprintf(stderr,
548 "\n"
549 "--help Prints this information.\n"
550 "\n"
551 "--dir=<dirname> The directory containing the routing database.\n"
552 "--prefix=<name> The filename prefix for the routing database.\n"
553 "\n"
554 "--sort-ram-size=<size> The amount of RAM (in MB) to use for data sorting\n"
555 #if SLIM
556 " (defaults to 64MB otherwise.)\n"
557 #else
558 " (defaults to 256MB otherwise.)\n"
559 #endif
560 #if defined(USE_PTHREADS) && USE_PTHREADS
561 "--sort-threads=<number> The number of threads to use for data sorting.\n"
562 #endif
563 "\n"
564 "--tmpdir=<dirname> The directory name for temporary files.\n"
565 " (defaults to the '--dir' option directory.)\n"
566 "\n"
567 "--tagging=<filename> The name of the XML file containing the tagging rules\n"
568 " (defaults to 'tagging.xml' with '--dir' and\n"
569 " '--prefix' options or the file installed in\n"
570 " '" DATADIR "').\n"
571 "\n"
572 "--loggable Print progress messages suitable for logging to file.\n"
573 "--logtime Print the elapsed time for each processing step.\n"
574 "--errorlog[=<name>] Log parsing errors to 'error.log' or the given name\n"
575 " (the '--dir' and '--prefix' options are applied).\n"
576 "\n"
577 "--parse-only Parse the OSM file(s) and store the results.\n"
578 "--process-only Process the stored results from previous option.\n"
579 "--append Parse the OSM file(s) and append to existing results.\n"
580 "--preserve Keep the intermediate files after parsing & sorting.\n"
581 "\n"
582 "--max-iterations=<number> The number of iterations for finding super-nodes\n"
583 " (defaults to 5).\n"
584 "\n"
585 "--prune-none Disable the prune options below, they are re-enabled\n"
586 " by adding them to the command line after this option.\n"
587 "--prune-isolated=<len> Remove access from small disconnected segment groups\n"
588 " (defaults to removing groups under 500m).\n"
589 "--prune-short=<len> Remove short segments (defaults to removing segments\n"
590 " up to a maximum length of 5m).\n"
591 "--prune-straight=<len> Remove nodes in almost straight highways (defaults to\n"
592 " removing nodes up to 3m offset from a straight line).\n"
593 "\n"
594 "<filename.osm> ... The name(s) of the file(s) to process (by default\n"
595 " data is read from standard input).\n"
596 "\n"
597 "<transport> defaults to all but can be set to:\n"
598 "%s"
599 "\n"
600 "<highway> can be selected from:\n"
601 "%s"
602 "\n"
603 "<property> can be selected from:\n"
604 "%s",
605 TransportList(),HighwayList(),PropertyList());
606
607 exit(!detail);
608 }

Properties

Name Value
cvs:description Planet file splitter.
Index: This Site Routino Software Download Source Code Documentation XML Files Links
Index: View SVN Development Recent changes Development (destination-access branch) Version 3.4.2 (destination-access branch) Version 3.4.2 Version 3.4.1 (destination-access branch) Version 3.4.1 Version 3.4 (destination-access branch) Version 3.4 Version 3.3.3 (destination-access branch) Version 3.3.3 Version 3.3.2 (destination-access branch) Version 3.3.2 Version 3.3.1 (destination-access branch) Version 3.3.1 Version 3.3 (destination-access branch) Version 3.3 Version 3.2 Version 3.1.1 Version 3.1 Version 3.0 Version 2.7.3 Version 2.7.2 Version 2.7.1 Version 2.7 Version 2.6 Version 2.5.1 Version 2.5 Version 2.4.1 Version 2.4 Version 2.3.2 Version 2.3.1 Version 2.3 Version 2.2 Version 2.1.2 Version 2.1.1 Version 2.1 Version 2.0.3 Version 2.0.2 Version 2.0.1 Version 2.0 Version 1.5.1 Version 1.5 Version 1.4.1 Version 1.4 Version 1.3 Version 1.2 Version 1.1 Version 1.0
Powered by ViewVC 1.3.0-dev ViewVC Help
© Andrew M. Bishop = < amb @ ... >
Root