Routino SVN Repository Browser

Check out the latest version of Routino: svn co http://routino.org/svn/trunk routino

/[routino]/trunk/src/planetsplitter.c
ViewVC logotype

Contents of /trunk/src/planetsplitter.c

Parent Directory Parent Directory | Revision Log Revision Log

Revision 1120 - (show annotations) (download) (as text)
Thu Nov 1 20:00:27 2012 UTC (12 years, 5 months ago) by amb
File MIME type: text/x-csrc
File size: 17659 byte(s) 
Introduce a new'--append' option for appending data from a file to the currently
parsed data.  Rename the intermediate file used for storing data to be appended
to.  Add a function to call after appending to a file which closes the file and
renames it to a temporary filename which is used for the remaining processing.
1 /***************************************
2 OSM planet file splitter.
3
4 Part of the Routino routing software.
5 ******************/ /******************
6 This file Copyright 2008-2012 Andrew M. Bishop
7
8 This program is free software: you can redistribute it and/or modify
9 it under the terms of the GNU Affero General Public License as published by
10 the Free Software Foundation, either version 3 of the License, or
11 (at your option) any later version.
12
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU Affero General Public License for more details.
17
18 You should have received a copy of the GNU Affero General Public License
19 along with this program. If not, see <http://www.gnu.org/licenses/>.
20 ***************************************/
21
22
23 #include <stdio.h>
24 #include <stdlib.h>
25 #include <string.h>
26 #include <errno.h>
27 #include <sys/time.h>
28
29 #include "types.h"
30 #include "ways.h"
31
32 #include "typesx.h"
33 #include "nodesx.h"
34 #include "segmentsx.h"
35 #include "waysx.h"
36 #include "relationsx.h"
37 #include "superx.h"
38 #include "prunex.h"
39
40 #include "files.h"
41 #include "logging.h"
42 #include "functions.h"
43 #include "osmparser.h"
44 #include "tagging.h"
45
46
47 /* Global variables */
48
49 /*+ The name of the temporary directory. +*/
50 char *option_tmpdirname=NULL;
51
52 /*+ The amount of RAM to use for filesorting. +*/
53 size_t option_filesort_ramsize=0;
54
55 /*+ The number of threads to use for filesorting. +*/
56 int option_filesort_threads=1;
57
58
59 /* Local functions */
60
61 static void print_usage(int detail,const char *argerr,const char *err);
62
63
64 /*++++++++++++++++++++++++++++++++++++++
65 The main program for the planetsplitter.
66 ++++++++++++++++++++++++++++++++++++++*/
67
68 int main(int argc,char** argv)
69 {
70 struct timeval start_time;
71 NodesX *Nodes;
72 SegmentsX *Segments,*SuperSegments=NULL,*MergedSegments=NULL;
73 WaysX *Ways;
74 RelationsX *Relations;
75 int iteration=0,quit=0;
76 int max_iterations=5;
77 char *dirname=NULL,*prefix=NULL,*tagging=NULL,*errorlog=NULL;
78 int option_parse_only=0,option_process_only=0,option_append=0;
79 int option_filenames=0;
80 int option_prune_isolated=500,option_prune_short=5,option_prune_straight=3;
81 int arg;
82
83 gettimeofday(&start_time,NULL);
84
85 /* Parse the command line arguments */
86
87 for(arg=1;arg<argc;arg++)
88 {
89 if(!strcmp(argv[arg],"--help"))
90 print_usage(1,NULL,NULL);
91 else if(!strncmp(argv[arg],"--sort-ram-size=",16))
92 option_filesort_ramsize=atoi(&argv[arg][16]);
93 #if defined(USE_PTHREADS) && USE_PTHREADS
94 else if(!strncmp(argv[arg],"--sort-threads=",15))
95 option_filesort_threads=atoi(&argv[arg][15]);
96 #endif
97 else if(!strncmp(argv[arg],"--dir=",6))
98 dirname=&argv[arg][6];
99 else if(!strncmp(argv[arg],"--tmpdir=",9))
100 option_tmpdirname=&argv[arg][9];
101 else if(!strncmp(argv[arg],"--prefix=",9))
102 prefix=&argv[arg][9];
103 else if(!strcmp(argv[arg],"--parse-only"))
104 option_parse_only=1;
105 else if(!strcmp(argv[arg],"--process-only"))
106 option_process_only=1;
107 else if(!strcmp(argv[arg],"--append"))
108 option_append=1;
109 else if(!strcmp(argv[arg],"--loggable"))
110 option_loggable=1;
111 else if(!strcmp(argv[arg],"--logtime"))
112 option_logtime=1;
113 else if(!strcmp(argv[arg],"--errorlog"))
114 errorlog="error.log";
115 else if(!strncmp(argv[arg],"--errorlog=",11))
116 errorlog=&argv[arg][11];
117 else if(!strncmp(argv[arg],"--max-iterations=",17))
118 max_iterations=atoi(&argv[arg][17]);
119 else if(!strncmp(argv[arg],"--tagging=",10))
120 tagging=&argv[arg][10];
121 else if(!strncmp(argv[arg],"--prune",7))
122 {
123 if(!strcmp(&argv[arg][7],"-none"))
124 option_prune_isolated=option_prune_short=option_prune_straight=0;
125 else if(!strncmp(&argv[arg][7],"-isolated=",10))
126 option_prune_isolated=atoi(&argv[arg][17]);
127 else if(!strncmp(&argv[arg][7],"-short=",7))
128 option_prune_short=atoi(&argv[arg][14]);
129 else if(!strncmp(&argv[arg][7],"-straight=",10))
130 option_prune_straight=atoi(&argv[arg][17]);
131 else
132 print_usage(0,argv[arg],NULL);
133 }
134 else if(argv[arg][0]=='-' && argv[arg][1]=='-')
135 print_usage(0,argv[arg],NULL);
136 else
137 option_filenames++;
138 }
139
140 /* Check the specified command line options */
141
142 if(option_parse_only && option_process_only)
143 print_usage(0,NULL,"Cannot use '--parse-only' and '--process-only' at the same time.");
144
145 if(option_append && option_process_only)
146 print_usage(0,NULL,"Cannot use '--append' and '--process-only' at the same time.");
147
148 if(option_filenames && option_process_only)
149 print_usage(0,NULL,"Cannot use '--process-only' and filenames at the same time.");
150
151 if(!option_filesort_ramsize)
152 {
153 #if SLIM
154 option_filesort_ramsize=64*1024*1024;
155 #else
156 option_filesort_ramsize=256*1024*1024;
157 #endif
158 }
159 else
160 option_filesort_ramsize*=1024*1024;
161
162 if(!option_tmpdirname)
163 {
164 if(!dirname)
165 option_tmpdirname=".";
166 else
167 option_tmpdirname=dirname;
168 }
169
170
171 if(!option_process_only)
172 {
173 if(tagging)
174 {
175 if(!ExistsFile(tagging))
176 {
177 fprintf(stderr,"Error: The '--tagging' option specifies a file that does not exist.\n");
178 return(1);
179 }
180 }
181 else
182 {
183 if(ExistsFile(FileName(dirname,prefix,"tagging.xml")))
184 tagging=FileName(dirname,prefix,"tagging.xml");
185 else if(ExistsFile(FileName(DATADIR,NULL,"tagging.xml")))
186 tagging=FileName(DATADIR,NULL,"tagging.xml");
187 else
188 {
189 fprintf(stderr,"Error: The '--tagging' option was not used and the default 'tagging.xml' does not exist.\n");
190 return(1);
191 }
192 }
193
194 if(ParseXMLTaggingRules(tagging))
195 {
196 fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging);
197 return(1);
198 }
199 }
200
201 /* Create new node, segment, way and relation variables */
202
203 Nodes=NewNodeList(option_append||option_process_only);
204
205 Segments=NewSegmentList(option_append||option_process_only);
206
207 Ways=NewWayList(option_append||option_process_only);
208
209 Relations=NewRelationList(option_append||option_process_only);
210
211 /* Create the error log file */
212
213 if(errorlog)
214 open_errorlog(FileName(dirname,prefix,errorlog),option_append||option_process_only);
215
216 /* Parse the file */
217
218 if(!option_process_only)
219 {
220 if(option_filenames)
221 {
222 for(arg=1;arg<argc;arg++)
223 {
224 FILE *file;
225
226 if(argv[arg][0]=='-' && argv[arg][1]=='-')
227 continue;
228
229 file=fopen(argv[arg],"rb");
230
231 if(!file)
232 {
233 fprintf(stderr,"Cannot open file '%s' for reading [%s].\n",argv[arg],strerror(errno));
234 exit(EXIT_FAILURE);
235 }
236
237 printf("\nParse OSM Data [%s]\n==============\n\n",argv[arg]);
238 fflush(stdout);
239
240 if(ParseOSM(file,Nodes,Segments,Ways,Relations))
241 exit(EXIT_FAILURE);
242
243 fclose(file);
244 }
245 }
246 else
247 {
248 printf("\nParse OSM Data\n==============\n\n");
249 fflush(stdout);
250
251 if(ParseOSM(stdin,Nodes,Segments,Ways,Relations))
252 exit(EXIT_FAILURE);
253 }
254
255 DeleteXMLTaggingRules();
256 }
257
258 if(option_parse_only)
259 {
260 FreeNodeList(Nodes,1);
261 FreeSegmentList(Segments,1);
262 FreeWayList(Ways,1);
263 FreeRelationList(Relations,1);
264
265 return(0);
266 }
267
268 FinishNodeList(Nodes);
269 FinishSegmentList(Segments);
270 FinishWayList(Ways);
271 FinishRelationList(Relations);
272
273 /* Process the data */
274
275 printf("\nProcess OSM Data\n================\n\n");
276 fflush(stdout);
277
278 /* Sort the nodes, segments, ways and relations */
279
280 SortNodeList(Nodes);
281
282 SortSegmentList(Segments);
283
284 SortWayList(Ways);
285
286 SortRelationList(Relations);
287
288 /* Remove bad segments (must be after sorting the nodes and segments) */
289
290 RemoveBadSegments(Nodes,Segments);
291
292 /* Remove non-highway nodes (must be after removing the bad segments) */
293
294 RemoveNonHighwayNodes(Nodes,Segments);
295
296 /* Process the route relations and first part of turn relations (must be before compacting the ways) */
297
298 ProcessRouteRelations(Relations,Ways);
299
300 ProcessTurnRelations1(Relations,Nodes,Ways);
301
302 /* Measure the segments and replace node/way id with index (must be after removing non-highway nodes) */
303
304 MeasureSegments(Segments,Nodes,Ways);
305
306 /* Index the segments */
307
308 IndexSegments(Segments,Nodes,Ways);
309
310 /* Convert the turn relations from ways into nodes */
311
312 ProcessTurnRelations2(Relations,Nodes,Segments,Ways);
313
314 /* Compact the ways (must be after turn relations 2) */
315
316 CompactWayList(Ways,Segments);
317
318 /* Index the segments */
319
320 IndexSegments(Segments,Nodes,Ways);
321
322 /* Prune unwanted nodes/segments. */
323
324 if(option_prune_straight || option_prune_isolated || option_prune_short)
325 {
326 printf("\nPrune Unneeded Data\n===================\n\n");
327 fflush(stdout);
328
329 StartPruning(Nodes,Segments,Ways);
330
331 if(option_prune_straight)
332 PruneStraightHighwayNodes(Nodes,Segments,Ways,option_prune_straight);
333
334 if(option_prune_isolated)
335 PruneIsolatedRegions(Nodes,Segments,Ways,option_prune_isolated);
336
337 if(option_prune_short)
338 PruneShortSegments(Nodes,Segments,Ways,option_prune_short);
339
340 FinishPruning(Nodes,Segments,Ways);
341
342 /* Remove the pruned nodes and segments and update the indexes */
343
344 RemovePrunedNodes(Nodes,Segments);
345 RemovePrunedSegments(Segments,Ways);
346 CompactWayList(Ways,Segments);
347 RemovePrunedTurnRelations(Relations,Nodes);
348 IndexSegments(Segments,Nodes,Ways);
349 }
350
351 /* Repeated iteration on Super-Nodes and Super-Segments */
352
353 do
354 {
355 int nsuper;
356
357 printf("\nProcess Super-Data (iteration %d)\n================================%s\n\n",iteration,iteration>9?"=":"");
358 fflush(stdout);
359
360 if(iteration==0)
361 {
362 /* Select the super-nodes */
363
364 ChooseSuperNodes(Nodes,Segments,Ways);
365
366 /* Select the super-segments */
367
368 SuperSegments=CreateSuperSegments(Nodes,Segments,Ways);
369
370 nsuper=Segments->number;
371 }
372 else
373 {
374 SegmentsX *SuperSegments2;
375
376 /* Select the super-nodes */
377
378 ChooseSuperNodes(Nodes,SuperSegments,Ways);
379
380 /* Select the super-segments */
381
382 SuperSegments2=CreateSuperSegments(Nodes,SuperSegments,Ways);
383
384 nsuper=SuperSegments->number;
385
386 FreeSegmentList(SuperSegments,0);
387
388 SuperSegments=SuperSegments2;
389 }
390
391 /* Sort the super-segments */
392
393 SortSegmentList(SuperSegments);
394
395 /* Remove duplicated super-segments */
396
397 DeduplicateSegments(SuperSegments,Nodes,Ways);
398
399 /* Index the segments */
400
401 IndexSegments(SuperSegments,Nodes,Ways);
402
403 /* Check for end condition */
404
405 if(SuperSegments->number==nsuper)
406 quit=1;
407
408 iteration++;
409
410 if(iteration>max_iterations)
411 quit=1;
412 }
413 while(!quit);
414
415 /* Combine the super-segments */
416
417 printf("\nCombine Segments and Super-Segments\n===================================\n\n");
418 fflush(stdout);
419
420 /* Merge the super-segments */
421
422 MergedSegments=MergeSuperSegments(Segments,SuperSegments);
423
424 FreeSegmentList(Segments,0);
425
426 FreeSegmentList(SuperSegments,0);
427
428 Segments=MergedSegments;
429
430 /* Sort and re-index the merged segments */
431
432 SortSegmentList(Segments);
433
434 IndexSegments(Segments,Nodes,Ways);
435
436 /* Cross reference the nodes and segments */
437
438 printf("\nCross-Reference Nodes and Segments\n==================================\n\n");
439 fflush(stdout);
440
441 /* Sort the nodes and segments geographically */
442
443 SortNodeListGeographically(Nodes);
444
445 SortSegmentListGeographically(Segments,Nodes);
446
447 /* Re-index the segments */
448
449 IndexSegments(Segments,Nodes,Ways);
450
451 /* Sort the turn relations geographically */
452
453 SortTurnRelationListGeographically(Relations,Nodes,Segments);
454
455 /* Output the results */
456
457 printf("\nWrite Out Database Files\n========================\n\n");
458 fflush(stdout);
459
460 /* Write out the nodes */
461
462 SaveNodeList(Nodes,FileName(dirname,prefix,"nodes.mem"),Segments);
463
464 FreeNodeList(Nodes,0);
465
466 /* Write out the segments */
467
468 SaveSegmentList(Segments,FileName(dirname,prefix,"segments.mem"));
469
470 FreeSegmentList(Segments,0);
471
472 /* Write out the ways */
473
474 SaveWayList(Ways,FileName(dirname,prefix,"ways.mem"));
475
476 FreeWayList(Ways,0);
477
478 /* Write out the relations */
479
480 SaveRelationList(Relations,FileName(dirname,prefix,"relations.mem"));
481
482 FreeRelationList(Relations,0);
483
484 /* Close the error log file */
485
486 if(errorlog)
487 close_errorlog();
488
489 /* Print the total time */
490
491 if(option_logtime)
492 {
493 printf("\n");
494 fprintf_elapsed_time(stdout,&start_time);
495 printf("Complete\n");
496 fflush(stdout);
497 }
498
499 return(0);
500 }
501
502
503 /*++++++++++++++++++++++++++++++++++++++
504 Print out the usage information.
505
506 int detail The level of detail to use - 0 = low, 1 = high.
507
508 const char *argerr The argument that gave the error (if there is one).
509
510 const char *err Other error message (if there is one).
511 ++++++++++++++++++++++++++++++++++++++*/
512
513 static void print_usage(int detail,const char *argerr,const char *err)
514 {
515 fprintf(stderr,
516 "Usage: planetsplitter [--help]\n"
517 " [--dir=<dirname>] [--prefix=<name>]\n"
518 #if defined(USE_PTHREADS) && USE_PTHREADS
519 " [--sort-ram-size=<size>] [--sort-threads=<number>]\n"
520 #else
521 " [--sort-ram-size=<size>]\n"
522 #endif
523 " [--tmpdir=<dirname>]\n"
524 " [--tagging=<filename>]\n"
525 " [--loggable] [--logtime]\n"
526 " [--errorlog[=<name>]]\n"
527 " [--parse-only | --process-only]\n"
528 " [--append]\n"
529 " [--max-iterations=<number>]\n"
530 " [--prune-none]\n"
531 " [--prune-isolated=<len>]\n"
532 " [--prune-short=<len>]\n"
533 " [--prune-straight=<len>]\n"
534 " [<filename.osm> ...]\n");
535
536 if(argerr)
537 fprintf(stderr,
538 "\n"
539 "Error with command line parameter: %s\n",argerr);
540
541 if(err)
542 fprintf(stderr,
543 "\n"
544 "Error: %s\n",err);
545
546 if(detail)
547 fprintf(stderr,
548 "\n"
549 "--help Prints this information.\n"
550 "\n"
551 "--dir=<dirname> The directory containing the routing database.\n"
552 "--prefix=<name> The filename prefix for the routing database.\n"
553 "\n"
554 "--sort-ram-size=<size> The amount of RAM (in MB) to use for data sorting\n"
555 #if SLIM
556 " (defaults to 64MB otherwise.)\n"
557 #else
558 " (defaults to 256MB otherwise.)\n"
559 #endif
560 #if defined(USE_PTHREADS) && USE_PTHREADS
561 "--sort-threads=<number> The number of threads to use for data sorting.\n"
562 #endif
563 "\n"
564 "--tmpdir=<dirname> The directory name for temporary files.\n"
565 " (defaults to the '--dir' option directory.)\n"
566 "\n"
567 "--tagging=<filename> The name of the XML file containing the tagging rules\n"
568 " (defaults to 'tagging.xml' with '--dir' and\n"
569 " '--prefix' options or the file installed in\n"
570 " '" DATADIR "').\n"
571 "\n"
572 "--loggable Print progress messages suitable for logging to file.\n"
573 "--logtime Print the elapsed time for each processing step.\n"
574 "--errorlog[=<name>] Log parsing errors to 'error.log' or the given name\n"
575 " (the '--dir' and '--prefix' options are applied).\n"
576 "\n"
577 "--parse-only Parse the OSM file(s) and store the results.\n"
578 "--process-only Process the stored results from previous option.\n"
579 "--append Parse the OSM file(s) and append to existing results.\n"
580 "\n"
581 "--max-iterations=<number> The number of iterations for finding super-nodes\n"
582 " (defaults to 5).\n"
583 "\n"
584 "--prune-none Disable the prune options below, they are re-enabled\n"
585 " by adding them to the command line after this option.\n"
586 "--prune-isolated=<len> Remove access from small disconnected segment groups\n"
587 " (defaults to removing groups under 500m).\n"
588 "--prune-short=<len> Remove short segments (defaults to removing segments\n"
589 " up to a maximum length of 5m).\n"
590 "--prune-straight=<len> Remove nodes in almost straight highways (defaults to\n"
591 " removing nodes up to 3m offset from a straight line).\n"
592 "\n"
593 "<filename.osm> ... The name(s) of the file(s) to process (by default\n"
594 " data is read from standard input).\n"
595 "\n"
596 "<transport> defaults to all but can be set to:\n"
597 "%s"
598 "\n"
599 "<highway> can be selected from:\n"
600 "%s"
601 "\n"
602 "<property> can be selected from:\n"
603 "%s",
604 TransportList(),HighwayList(),PropertyList());
605
606 exit(!detail);
607 }

Properties

Name Value
cvs:description Planet file splitter.
Index: This Site Routino Software Download Source Code Documentation XML Files Links
Index: View SVN Development Recent changes Development (destination-access branch) Version 3.4.2 (destination-access branch) Version 3.4.2 Version 3.4.1 (destination-access branch) Version 3.4.1 Version 3.4 (destination-access branch) Version 3.4 Version 3.3.3 (destination-access branch) Version 3.3.3 Version 3.3.2 (destination-access branch) Version 3.3.2 Version 3.3.1 (destination-access branch) Version 3.3.1 Version 3.3 (destination-access branch) Version 3.3 Version 3.2 Version 3.1.1 Version 3.1 Version 3.0 Version 2.7.3 Version 2.7.2 Version 2.7.1 Version 2.7 Version 2.6 Version 2.5.1 Version 2.5 Version 2.4.1 Version 2.4 Version 2.3.2 Version 2.3.1 Version 2.3 Version 2.2 Version 2.1.2 Version 2.1.1 Version 2.1 Version 2.0.3 Version 2.0.2 Version 2.0.1 Version 2.0 Version 1.5.1 Version 1.5 Version 1.4.1 Version 1.4 Version 1.3 Version 1.2 Version 1.1 Version 1.0
Powered by ViewVC 1.3.0-dev ViewVC Help
© Andrew M. Bishop = < amb @ ... >
Root