Routino SVN Repository Browser

Check out the latest version of Routino: svn co http://routino.org/svn/trunk routino

ViewVC logotype

Contents of /trunk/src/nodesx.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1136 - (show annotations) (download) (as text)
Sat Nov 10 19:23:32 2012 UTC (12 years, 4 months ago) by amb
File MIME type: text/x-csrc
File size: 18236 byte(s)
Added a --preserve option which keeps the raw data files after parsing, sorting
and de-duplication.

1 /***************************************
2 Extented Node data type functions.
3
4 Part of the Routino routing software.
5 ******************/ /******************
6 This file Copyright 2008-2012 Andrew M. Bishop
7
8 This program is free software: you can redistribute it and/or modify
9 it under the terms of the GNU Affero General Public License as published by
10 the Free Software Foundation, either version 3 of the License, or
11 (at your option) any later version.
12
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU Affero General Public License for more details.
17
18 You should have received a copy of the GNU Affero General Public License
19 along with this program. If not, see <http://www.gnu.org/licenses/>.
20 ***************************************/
21
22
23 #include <assert.h>
24 #include <stdlib.h>
25 #include <string.h>
26
27 #include "types.h"
28 #include "nodes.h"
29
30 #include "typesx.h"
31 #include "nodesx.h"
32 #include "segmentsx.h"
33 #include "waysx.h"
34
35 #include "files.h"
36 #include "logging.h"
37 #include "sorting.h"
38
39
40 /* Variables */
41
42 /*+ The command line '--tmpdir' option or its default value. +*/
43 extern char *option_tmpdirname;
44
45 /*+ Temporary file-local variables for use by the sort functions. +*/
46 static NodesX *sortnodesx;
47 static latlong_t lat_min,lat_max,lon_min,lon_max;
48
49 /* Functions */
50
51 static int sort_by_id(NodeX *a,NodeX *b);
52 static int deduplicate_and_index_by_id(NodeX *nodex,index_t index);
53
54 static int sort_by_lat_long(NodeX *a,NodeX *b);
55 static int index_by_lat_long(NodeX *nodex,index_t index);
56
57
58 /*++++++++++++++++++++++++++++++++++++++
59 Allocate a new node list (create a new file or open an existing one).
60
61 NodesX *NewNodeList Returns a pointer to the node list.
62
63 int append Set to 1 if the file is to be opened for appending.
64
65 int readonly Set to 1 if the file is not to be opened.
66 ++++++++++++++++++++++++++++++++++++++*/
67
68 NodesX *NewNodeList(int append,int readonly)
69 {
70 NodesX *nodesx;
71
72 nodesx=(NodesX*)calloc(1,sizeof(NodesX));
73
74 assert(nodesx); /* Check calloc() worked */
75
76 nodesx->filename =(char*)malloc(strlen(option_tmpdirname)+32);
77 nodesx->filename_tmp=(char*)malloc(strlen(option_tmpdirname)+32);
78
79 sprintf(nodesx->filename ,"%s/nodesx.parsed.mem",option_tmpdirname);
80 sprintf(nodesx->filename_tmp,"%s/nodesx.%p.tmp" ,option_tmpdirname,(void*)nodesx);
81
82 if(append || readonly)
83 if(ExistsFile(nodesx->filename))
84 {
85 off_t size;
86
87 size=SizeFile(nodesx->filename);
88
89 nodesx->number=size/sizeof(NodeX);
90 }
91
92 if(append)
93 nodesx->fd=OpenFileAppend(nodesx->filename);
94 else if(!readonly)
95 nodesx->fd=OpenFileNew(nodesx->filename);
96 else
97 nodesx->fd=-1;
98
99 return(nodesx);
100 }
101
102
103 /*++++++++++++++++++++++++++++++++++++++
104 Free a node list.
105
106 NodesX *nodesx The set of nodes to be freed.
107 ++++++++++++++++++++++++++++++++++++++*/
108
109 void FreeNodeList(NodesX *nodesx)
110 {
111 DeleteFile(nodesx->filename_tmp);
112
113 free(nodesx->filename);
114 free(nodesx->filename_tmp);
115
116 if(nodesx->idata)
117 free(nodesx->idata);
118
119 if(nodesx->gdata)
120 free(nodesx->gdata);
121
122 if(nodesx->pdata)
123 free(nodesx->pdata);
124
125 if(nodesx->super)
126 free(nodesx->super);
127
128 free(nodesx);
129 }
130
131
132 /*++++++++++++++++++++++++++++++++++++++
133 Append a single node to an unsorted node list.
134
135 NodesX *nodesx The set of nodes to modify.
136
137 node_t id The node identifier from the original OSM data.
138
139 double latitude The latitude of the node.
140
141 double longitude The longitude of the node.
142
143 transports_t allow The allowed traffic types through the node.
144
145 uint16_t flags The flags to set for this node.
146 ++++++++++++++++++++++++++++++++++++++*/
147
148 void AppendNode(NodesX *nodesx,node_t id,double latitude,double longitude,transports_t allow,uint16_t flags)
149 {
150 NodeX nodex;
151
152 nodex.id=id;
153 nodex.latitude =radians_to_latlong(latitude);
154 nodex.longitude=radians_to_latlong(longitude);
155 nodex.allow=allow;
156 nodex.flags=flags;
157
158 WriteFile(nodesx->fd,&nodex,sizeof(NodeX));
159
160 nodesx->number++;
161
162 assert(nodesx->number<NODE_FAKE); /* NODE_FAKE marks the high-water mark for real nodes. */
163 }
164
165
166 /*++++++++++++++++++++++++++++++++++++++
167 Finish appending nodes and change the filename over.
168
169 NodesX *nodesx The nodes that have been appended.
170 ++++++++++++++++++++++++++++++++++++++*/
171
172 void FinishNodeList(NodesX *nodesx)
173 {
174 /* Close the file (finished appending) */
175
176 if(nodesx->fd!=-1)
177 nodesx->fd=CloseFile(nodesx->fd);
178
179 /* Rename the file to the temporary name (used everywhere else) */
180
181 RenameFile(nodesx->filename,nodesx->filename_tmp);
182 }
183
184
185 /*++++++++++++++++++++++++++++++++++++++
186 Sort the node list.
187
188 NodesX *nodesx The set of nodes to modify.
189 ++++++++++++++++++++++++++++++++++++++*/
190
191 void SortNodeList(NodesX *nodesx)
192 {
193 int fd;
194 index_t xnumber;
195
196 /* Print the start message */
197
198 printf_first("Sorting Nodes");
199
200 /* Re-open the file read-only and a new file writeable */
201
202 nodesx->fd=ReOpenFile(nodesx->filename_tmp);
203
204 DeleteFile(nodesx->filename_tmp);
205
206 fd=OpenFileNew(nodesx->filename_tmp);
207
208 /* Allocate the array of indexes */
209
210 nodesx->idata=(node_t*)malloc(nodesx->number*sizeof(node_t));
211
212 assert(nodesx->idata); /* Check malloc() worked */
213
214 /* Sort the nodes by ID and index them */
215
216 xnumber=nodesx->number;
217
218 sortnodesx=nodesx;
219
220 nodesx->number=filesort_fixed(nodesx->fd,fd,sizeof(NodeX),NULL,
221 (int (*)(const void*,const void*))sort_by_id,
222 (int (*)(void*,index_t))deduplicate_and_index_by_id);
223
224 /* Close the files */
225
226 nodesx->fd=CloseFile(nodesx->fd);
227 CloseFile(fd);
228
229 /* Print the final message */
230
231 printf_last("Sorted Nodes: Nodes=%"Pindex_t" Duplicates=%"Pindex_t,xnumber,xnumber-nodesx->number);
232 }
233
234
235 /*++++++++++++++++++++++++++++++++++++++
236 Sort the nodes into id order.
237
238 int sort_by_id Returns the comparison of the id fields.
239
240 NodeX *a The first extended node.
241
242 NodeX *b The second extended node.
243 ++++++++++++++++++++++++++++++++++++++*/
244
245 static int sort_by_id(NodeX *a,NodeX *b)
246 {
247 node_t a_id=a->id;
248 node_t b_id=b->id;
249
250 if(a_id<b_id)
251 return(-1);
252 else if(a_id>b_id)
253 return(1);
254 else
255 return(FILESORT_PRESERVE_ORDER(a,b));
256 }
257
258
259 /*++++++++++++++++++++++++++++++++++++++
260 Create the index of identifiers and discard duplicate nodes.
261
262 int deduplicate_and_index_by_id Return 1 if the value is to be kept, otherwise 0.
263
264 NodeX *nodex The extended node.
265
266 index_t index The number of sorted nodes that have already been written to the output file.
267 ++++++++++++++++++++++++++++++++++++++*/
268
269 static int deduplicate_and_index_by_id(NodeX *nodex,index_t index)
270 {
271 if(index==0 || sortnodesx->idata[index-1]!=nodex->id)
272 {
273 sortnodesx->idata[index]=nodex->id;
274
275 return(1);
276 }
277 else
278 {
279 logerror("Node %"Pnode_t" is duplicated\n",nodex->id);
280
281 return(0);
282 }
283 }
284
285
286 /*++++++++++++++++++++++++++++++++++++++
287 Sort the node list geographically.
288
289 NodesX *nodesx The set of nodes to modify.
290 ++++++++++++++++++++++++++++++++++++++*/
291
292 void SortNodeListGeographically(NodesX *nodesx)
293 {
294 int fd;
295 ll_bin_t lat_min_bin,lat_max_bin,lon_min_bin,lon_max_bin;
296
297 /* While we are here we can work out the range of data */
298
299 lat_min=radians_to_latlong( 2);
300 lat_max=radians_to_latlong(-2);
301 lon_min=radians_to_latlong( 4);
302 lon_max=radians_to_latlong(-4);
303
304 /* Print the start message */
305
306 printf_first("Sorting Nodes Geographically");
307
308 /* Allocate the memory for the geographical index array */
309
310 nodesx->gdata=(index_t*)malloc(nodesx->number*sizeof(index_t));
311
312 assert(nodesx->gdata); /* Check malloc() worked */
313
314 /* Re-open the file read-only and a new file writeable */
315
316 nodesx->fd=ReOpenFile(nodesx->filename_tmp);
317
318 DeleteFile(nodesx->filename_tmp);
319
320 fd=OpenFileNew(nodesx->filename_tmp);
321
322 /* Sort nodes geographically and index them */
323
324 sortnodesx=nodesx;
325
326 filesort_fixed(nodesx->fd,fd,sizeof(NodeX),NULL,
327 (int (*)(const void*,const void*))sort_by_lat_long,
328 (int (*)(void*,index_t))index_by_lat_long);
329
330 /* Close the files */
331
332 nodesx->fd=CloseFile(nodesx->fd);
333 CloseFile(fd);
334
335 /* Free the memory */
336
337 free(nodesx->super);
338 nodesx->super=NULL;
339
340 /* Work out the number of bins */
341
342 lat_min_bin=latlong_to_bin(lat_min);
343 lon_min_bin=latlong_to_bin(lon_min);
344 lat_max_bin=latlong_to_bin(lat_max);
345 lon_max_bin=latlong_to_bin(lon_max);
346
347 nodesx->latzero=lat_min_bin;
348 nodesx->lonzero=lon_min_bin;
349
350 nodesx->latbins=(lat_max_bin-lat_min_bin)+1;
351 nodesx->lonbins=(lon_max_bin-lon_min_bin)+1;
352
353 /* Print the final message */
354
355 printf_last("Sorted Nodes Geographically: Nodes=%"Pindex_t,nodesx->number);
356 }
357
358
359 /*++++++++++++++++++++++++++++++++++++++
360 Sort the nodes into latitude and longitude order (first by longitude bin
361 number, then by latitude bin number and then by exact longitude and then by
362 exact latitude).
363
364 int sort_by_lat_long Returns the comparison of the latitude and longitude fields.
365
366 NodeX *a The first extended node.
367
368 NodeX *b The second extended node.
369 ++++++++++++++++++++++++++++++++++++++*/
370
371 static int sort_by_lat_long(NodeX *a,NodeX *b)
372 {
373 ll_bin_t a_lon=latlong_to_bin(a->longitude);
374 ll_bin_t b_lon=latlong_to_bin(b->longitude);
375
376 if(a_lon<b_lon)
377 return(-1);
378 else if(a_lon>b_lon)
379 return(1);
380 else
381 {
382 ll_bin_t a_lat=latlong_to_bin(a->latitude);
383 ll_bin_t b_lat=latlong_to_bin(b->latitude);
384
385 if(a_lat<b_lat)
386 return(-1);
387 else if(a_lat>b_lat)
388 return(1);
389 else
390 {
391 if(a->longitude<b->longitude)
392 return(-1);
393 else if(a->longitude>b->longitude)
394 return(1);
395 else
396 {
397 if(a->latitude<b->latitude)
398 return(-1);
399 else if(a->latitude>b->latitude)
400 return(1);
401 }
402
403 return(FILESORT_PRESERVE_ORDER(a,b));
404 }
405 }
406 }
407
408
409 /*++++++++++++++++++++++++++++++++++++++
410 Create the index between the sorted and unsorted nodes.
411
412 int index_by_lat_long Return 1 if the value is to be kept, otherwise 0.
413
414 NodeX *nodex The extended node.
415
416 index_t index The number of sorted nodes that have already been written to the output file.
417 ++++++++++++++++++++++++++++++++++++++*/
418
419 static int index_by_lat_long(NodeX *nodex,index_t index)
420 {
421 sortnodesx->gdata[nodex->id]=index;
422
423 if(IsBitSet(sortnodesx->super,nodex->id))
424 nodex->flags|=NODE_SUPER;
425
426 if(nodex->latitude<lat_min)
427 lat_min=nodex->latitude;
428 if(nodex->latitude>lat_max)
429 lat_max=nodex->latitude;
430 if(nodex->longitude<lon_min)
431 lon_min=nodex->longitude;
432 if(nodex->longitude>lon_max)
433 lon_max=nodex->longitude;
434
435 return(1);
436 }
437
438
439 /*++++++++++++++++++++++++++++++++++++++
440 Find a particular node index.
441
442 index_t IndexNodeX Returns the index of the extended node with the specified id.
443
444 NodesX *nodesx The set of nodes to use.
445
446 node_t id The node id to look for.
447 ++++++++++++++++++++++++++++++++++++++*/
448
449 index_t IndexNodeX(NodesX *nodesx,node_t id)
450 {
451 index_t start=0;
452 index_t end=nodesx->number-1;
453 index_t mid;
454
455 /* Binary search - search key exact match only is required.
456 *
457 * # <- start | Check mid and move start or end if it doesn't match
458 * # |
459 * # | Since an exact match is wanted we can set end=mid-1
460 * # <- mid | or start=mid+1 because we know that mid doesn't match.
461 * # |
462 * # | Eventually either end=start or end=start+1 and one of
463 * # <- end | start or end is the wanted one.
464 */
465
466 if(end<start) /* There are no nodes */
467 return(NO_NODE);
468 else if(id<nodesx->idata[start]) /* Check key is not before start */
469 return(NO_NODE);
470 else if(id>nodesx->idata[end]) /* Check key is not after end */
471 return(NO_NODE);
472 else
473 {
474 do
475 {
476 mid=(start+end)/2; /* Choose mid point */
477
478 if(nodesx->idata[mid]<id) /* Mid point is too low */
479 start=mid+1;
480 else if(nodesx->idata[mid]>id) /* Mid point is too high */
481 end=mid?(mid-1):mid;
482 else /* Mid point is correct */
483 return(mid);
484 }
485 while((end-start)>1);
486
487 if(nodesx->idata[start]==id) /* Start is correct */
488 return(start);
489
490 if(nodesx->idata[end]==id) /* End is correct */
491 return(end);
492 }
493
494 return(NO_NODE);
495 }
496
497
498 /*++++++++++++++++++++++++++++++++++++++
499 Remove any nodes that are not part of a highway.
500
501 NodesX *nodesx The set of nodes to modify.
502
503 SegmentsX *segmentsx The set of segments to use.
504
505 int preserve If set to 1 then keep the old data file otherwise delete it.
506 ++++++++++++++++++++++++++++++++++++++*/
507
508 void RemoveNonHighwayNodes(NodesX *nodesx,SegmentsX *segmentsx,int preserve)
509 {
510 NodeX nodex;
511 index_t total=0,highway=0,nothighway=0;
512 int fd;
513
514 /* Print the start message */
515
516 printf_first("Checking Nodes: Nodes=0");
517
518 /* Re-open the file read-only and a new file writeable */
519
520 nodesx->fd=ReOpenFile(nodesx->filename_tmp);
521
522 if(preserve)
523 RenameFile(nodesx->filename_tmp,nodesx->filename);
524 else
525 DeleteFile(nodesx->filename_tmp);
526
527 fd=OpenFileNew(nodesx->filename_tmp);
528
529 /* Modify the on-disk image */
530
531 while(!ReadFile(nodesx->fd,&nodex,sizeof(NodeX)))
532 {
533 if(!IsBitSet(segmentsx->usednode,total))
534 nothighway++;
535 else
536 {
537 nodex.id=highway;
538 nodesx->idata[highway]=nodesx->idata[total];
539
540 WriteFile(fd,&nodex,sizeof(NodeX));
541
542 highway++;
543 }
544
545 total++;
546
547 if(!(total%10000))
548 printf_middle("Checking Nodes: Nodes=%"Pindex_t" Highway=%"Pindex_t" not-Highway=%"Pindex_t,total,highway,nothighway);
549 }
550
551 nodesx->number=highway;
552
553 /* Close the files */
554
555 nodesx->fd=CloseFile(nodesx->fd);
556 CloseFile(fd);
557
558 /* Free the now-unneeded index */
559
560 free(segmentsx->usednode);
561 segmentsx->usednode=NULL;
562
563 /* Print the final message */
564
565 printf_last("Checked Nodes: Nodes=%"Pindex_t" Highway=%"Pindex_t" not-Highway=%"Pindex_t,total,highway,nothighway);
566 }
567
568
569 /*++++++++++++++++++++++++++++++++++++++
570 Remove any nodes that have been pruned.
571
572 NodesX *nodesx The set of nodes to prune.
573
574 SegmentsX *segmentsx The set of segments to use.
575 ++++++++++++++++++++++++++++++++++++++*/
576
577 void RemovePrunedNodes(NodesX *nodesx,SegmentsX *segmentsx)
578 {
579 NodeX nodex;
580 index_t total=0,pruned=0,notpruned=0;
581 int fd;
582
583 /* Print the start message */
584
585 printf_first("Deleting Pruned Nodes: Nodes=0 Pruned=0");
586
587 /* Allocate the array of indexes */
588
589 nodesx->pdata=(index_t*)malloc(nodesx->number*sizeof(index_t));
590
591 assert(nodesx->pdata); /* Check malloc() worked */
592
593 /* Re-open the file read-only and a new file writeable */
594
595 nodesx->fd=ReOpenFile(nodesx->filename_tmp);
596
597 DeleteFile(nodesx->filename_tmp);
598
599 fd=OpenFileNew(nodesx->filename_tmp);
600
601 /* Modify the on-disk image */
602
603 while(!ReadFile(nodesx->fd,&nodex,sizeof(NodeX)))
604 {
605 if(segmentsx->firstnode[total]==NO_SEGMENT)
606 {
607 pruned++;
608
609 nodesx->pdata[total]=NO_NODE;
610 }
611 else
612 {
613 nodex.id=notpruned;
614 nodesx->pdata[total]=notpruned;
615
616 WriteFile(fd,&nodex,sizeof(NodeX));
617
618 notpruned++;
619 }
620
621 total++;
622
623 if(!(total%10000))
624 printf_middle("Deleting Pruned Nodes: Nodes=%"Pindex_t" Pruned=%"Pindex_t,total,pruned);
625 }
626
627 nodesx->number=notpruned;
628
629 /* Close the files */
630
631 nodesx->fd=CloseFile(nodesx->fd);
632 CloseFile(fd);
633
634 /* Print the final message */
635
636 printf_last("Deleted Pruned Nodes: Nodes=%"Pindex_t" Pruned=%"Pindex_t,total,pruned);
637 }
638
639
640 /*++++++++++++++++++++++++++++++++++++++
641 Save the final node list database to a file.
642
643 NodesX *nodesx The set of nodes to save.
644
645 const char *filename The name of the file to save.
646
647 SegmentsX *segmentsx The set of segments to use.
648 ++++++++++++++++++++++++++++++++++++++*/
649
650 void SaveNodeList(NodesX *nodesx,const char *filename,SegmentsX *segmentsx)
651 {
652 index_t i;
653 int fd;
654 NodesFile nodesfile={0};
655 index_t super_number=0;
656 ll_bin2_t latlonbin=0,maxlatlonbins;
657 index_t *offsets;
658
659 /* Print the start message */
660
661 printf_first("Writing Nodes: Nodes=0");
662
663 /* Allocate the memory for the geographical offsets array */
664
665 offsets=(index_t*)malloc((nodesx->latbins*nodesx->lonbins+1)*sizeof(index_t));
666
667 assert(offsets); /* Check malloc() worked */
668
669 latlonbin=0;
670
671 /* Re-open the file */
672
673 nodesx->fd=ReOpenFile(nodesx->filename_tmp);
674
675 /* Write out the nodes data */
676
677 fd=OpenFileNew(filename);
678
679 SeekFile(fd,sizeof(NodesFile)+(nodesx->latbins*nodesx->lonbins+1)*sizeof(index_t));
680
681 for(i=0;i<nodesx->number;i++)
682 {
683 NodeX nodex;
684 Node node={0};
685 ll_bin_t latbin,lonbin;
686 ll_bin2_t llbin;
687
688 ReadFile(nodesx->fd,&nodex,sizeof(NodeX));
689
690 /* Create the Node */
691
692 node.latoffset=latlong_to_off(nodex.latitude);
693 node.lonoffset=latlong_to_off(nodex.longitude);
694 node.firstseg=segmentsx->firstnode[nodesx->gdata[nodex.id]];
695 node.allow=nodex.allow;
696 node.flags=nodex.flags;
697
698 if(node.flags&NODE_SUPER)
699 super_number++;
700
701 /* Work out the offsets */
702
703 latbin=latlong_to_bin(nodex.latitude )-nodesx->latzero;
704 lonbin=latlong_to_bin(nodex.longitude)-nodesx->lonzero;
705 llbin=lonbin*nodesx->latbins+latbin;
706
707 for(;latlonbin<=llbin;latlonbin++)
708 offsets[latlonbin]=i;
709
710 /* Write the data */
711
712 WriteFile(fd,&node,sizeof(Node));
713
714 if(!((i+1)%10000))
715 printf_middle("Writing Nodes: Nodes=%"Pindex_t,i+1);
716 }
717
718 /* Close the file */
719
720 nodesx->fd=CloseFile(nodesx->fd);
721
722 /* Finish off the offset indexing and write them out */
723
724 maxlatlonbins=nodesx->latbins*nodesx->lonbins;
725
726 for(;latlonbin<=maxlatlonbins;latlonbin++)
727 offsets[latlonbin]=nodesx->number;
728
729 SeekFile(fd,sizeof(NodesFile));
730 WriteFile(fd,offsets,(nodesx->latbins*nodesx->lonbins+1)*sizeof(index_t));
731
732 free(offsets);
733
734 /* Write out the header structure */
735
736 nodesfile.number=nodesx->number;
737 nodesfile.snumber=super_number;
738
739 nodesfile.latbins=nodesx->latbins;
740 nodesfile.lonbins=nodesx->lonbins;
741
742 nodesfile.latzero=nodesx->latzero;
743 nodesfile.lonzero=nodesx->lonzero;
744
745 SeekFile(fd,0);
746 WriteFile(fd,&nodesfile,sizeof(NodesFile));
747
748 CloseFile(fd);
749
750 /* Print the final message */
751
752 printf_last("Wrote Nodes: Nodes=%"Pindex_t,nodesx->number);
753 }

Properties

Name Value
cvs:description Extended nodes functions.