Routino SVN Repository Browser

Check out the latest version of Routino: svn co http://routino.org/svn/trunk routino

ViewVC logotype

Contents of /trunk/src/waysx.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1136 - (show annotations) (download) (as text)
Sat Nov 10 19:23:32 2012 UTC (12 years, 4 months ago) by amb
File MIME type: text/x-csrc
File size: 18035 byte(s)
Added a --preserve option which keeps the raw data files after parsing, sorting
and de-duplication.

1 /***************************************
2 Extended Way data type functions.
3
4 Part of the Routino routing software.
5 ******************/ /******************
6 This file Copyright 2008-2012 Andrew M. Bishop
7
8 This program is free software: you can redistribute it and/or modify
9 it under the terms of the GNU Affero General Public License as published by
10 the Free Software Foundation, either version 3 of the License, or
11 (at your option) any later version.
12
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU Affero General Public License for more details.
17
18 You should have received a copy of the GNU Affero General Public License
19 along with this program. If not, see <http://www.gnu.org/licenses/>.
20 ***************************************/
21
22
23 #include <assert.h>
24 #include <stdlib.h>
25 #include <string.h>
26
27 #include "types.h"
28 #include "ways.h"
29
30 #include "typesx.h"
31 #include "segmentsx.h"
32 #include "waysx.h"
33
34 #include "files.h"
35 #include "logging.h"
36 #include "sorting.h"
37
38
39 /* Global variables */
40
41 /*+ The command line '--tmpdir' option or its default value. +*/
42 extern char *option_tmpdirname;
43
44 /* Local variables */
45
46 /*+ Temporary file-local variables for use by the sort functions. +*/
47 static WaysX *sortwaysx;
48 static SegmentsX *sortsegmentsx;
49
50 /* Local functions */
51
52 static int sort_by_id(WayX *a,WayX *b);
53 static int sort_by_name(WayX *a,WayX *b);
54 static int sort_by_name_and_prop_and_id(WayX *a,WayX *b);
55
56 static int delete_unused(WayX *wayx,index_t index);
57 static int deduplicate_by_id(WayX *wayx,index_t index);
58 static int index_by_id(WayX *wayx,index_t index);
59 static int deduplicate_and_index_by_compact_id(WayX *wayx,index_t index);
60
61
62 /*++++++++++++++++++++++++++++++++++++++
63 Allocate a new way list (create a new file or open an existing one).
64
65 WaysX *NewWayList Returns the way list.
66
67 int append Set to 1 if the file is to be opened for appending.
68
69 int readonly Set to 1 if the file is not to be opened.
70 ++++++++++++++++++++++++++++++++++++++*/
71
72 WaysX *NewWayList(int append,int readonly)
73 {
74 WaysX *waysx;
75
76 waysx=(WaysX*)calloc(1,sizeof(WaysX));
77
78 assert(waysx); /* Check calloc() worked */
79
80 waysx->filename =(char*)malloc(strlen(option_tmpdirname)+32);
81 waysx->filename_tmp=(char*)malloc(strlen(option_tmpdirname)+32);
82
83 sprintf(waysx->filename ,"%s/waysx.parsed.mem",option_tmpdirname);
84 sprintf(waysx->filename_tmp,"%s/waysx.%p.tmp" ,option_tmpdirname,(void*)waysx);
85
86 if(append || readonly)
87 if(ExistsFile(waysx->filename))
88 {
89 off_t size,position=0;
90 int fd;
91
92 size=SizeFile(waysx->filename);
93
94 fd=ReOpenFile(waysx->filename);
95
96 while(position<size)
97 {
98 FILESORT_VARINT waysize;
99
100 SeekReadFile(fd,&waysize,FILESORT_VARSIZE,position);
101
102 waysx->number++;
103 position+=waysize+FILESORT_VARSIZE;
104 }
105
106 CloseFile(fd);
107 }
108
109 if(append)
110 waysx->fd=OpenFileAppend(waysx->filename);
111 else if(!readonly)
112 waysx->fd=OpenFileNew(waysx->filename);
113 else
114 waysx->fd=-1;
115
116
117 waysx->nfilename_tmp=(char*)malloc(strlen(option_tmpdirname)+32);
118
119 sprintf(waysx->nfilename_tmp,"%s/waynames.%p.tmp",option_tmpdirname,(void*)waysx);
120
121 return(waysx);
122 }
123
124
125 /*++++++++++++++++++++++++++++++++++++++
126 Free a way list.
127
128 WaysX *waysx The set of ways to be freed.
129 ++++++++++++++++++++++++++++++++++++++*/
130
131 void FreeWayList(WaysX *waysx)
132 {
133 DeleteFile(waysx->filename_tmp);
134
135 free(waysx->filename);
136 free(waysx->filename_tmp);
137
138 if(waysx->idata)
139 free(waysx->idata);
140
141 if(waysx->cdata)
142 free(waysx->cdata);
143
144 DeleteFile(waysx->nfilename_tmp);
145
146 free(waysx->nfilename_tmp);
147
148 free(waysx);
149 }
150
151
152 /*++++++++++++++++++++++++++++++++++++++
153 Append a single way to an unsorted way list.
154
155 WaysX *waysx The set of ways to process.
156
157 way_t id The ID of the way.
158
159 Way *way The way data itself.
160
161 const char *name The name or reference of the way.
162 ++++++++++++++++++++++++++++++++++++++*/
163
164 void AppendWay(WaysX *waysx,way_t id,Way *way,const char *name)
165 {
166 WayX wayx;
167 FILESORT_VARINT size;
168
169 wayx.id=id;
170 wayx.way=*way;
171
172 size=sizeof(WayX)+strlen(name)+1;
173
174 WriteFile(waysx->fd,&size,FILESORT_VARSIZE);
175 WriteFile(waysx->fd,&wayx,sizeof(WayX));
176 WriteFile(waysx->fd,name,strlen(name)+1);
177
178 waysx->number++;
179
180 assert(waysx->number!=0); /* Zero marks the high-water mark for ways. */
181 }
182
183
184 /*++++++++++++++++++++++++++++++++++++++
185 Finish appending ways and change the filename over.
186
187 WaysX *waysx The ways that have been appended.
188 ++++++++++++++++++++++++++++++++++++++*/
189
190 void FinishWayList(WaysX *waysx)
191 {
192 /* Close the file (finished appending) */
193
194 if(waysx->fd!=-1)
195 waysx->fd=CloseFile(waysx->fd);
196
197 /* Rename the file to the temporary name (used everywhere else) */
198
199 RenameFile(waysx->filename,waysx->filename_tmp);
200 }
201
202
203 /*++++++++++++++++++++++++++++++++++++++
204 Sort the list of ways.
205
206 WaysX *waysx The set of ways to process.
207 ++++++++++++++++++++++++++++++++++++++*/
208
209 void SortWayList(WaysX *waysx)
210 {
211 index_t xnumber;
212 int fd;
213
214 /* Print the start message */
215
216 printf_first("Sorting Ways");
217
218 /* Re-open the file read-only and a new file writeable */
219
220 waysx->fd=ReOpenFile(waysx->filename_tmp);
221
222 DeleteFile(waysx->filename_tmp);
223
224 fd=OpenFileNew(waysx->filename_tmp);
225
226 /* Sort the ways by ID and index them */
227
228 xnumber=waysx->number;
229
230 waysx->number=filesort_vary(waysx->fd,fd,NULL,
231 (int (*)(const void*,const void*))sort_by_id,
232 (int (*)(void*,index_t))deduplicate_by_id);
233
234 /* Close the files */
235
236 waysx->fd=CloseFile(waysx->fd);
237 CloseFile(fd);
238
239 /* Print the final message */
240
241 printf_last("Sorted Ways: Ways=%"Pindex_t" Duplicates=%"Pindex_t,xnumber,xnumber-waysx->number);
242 }
243
244
245 /*++++++++++++++++++++++++++++++++++++++
246 Extract the way names from the ways and reference the list of names from the ways.
247
248 WaysX *waysx The set of ways to process.
249
250 int preserve If set to 1 then keep the old data file otherwise delete it.
251 ++++++++++++++++++++++++++++++++++++++*/
252
253 void ExtractWayNames(WaysX *waysx,int preserve)
254 {
255 index_t i;
256 int fd;
257 char *names[2]={NULL,NULL};
258 int namelen[2]={0,0};
259 int nnames=0;
260 uint32_t lastlength=0;
261
262 /* Print the start message */
263
264 printf_first("Sorting Ways by Name");
265
266 /* Re-open the file read-only and a new file writeable */
267
268 waysx->fd=ReOpenFile(waysx->filename_tmp);
269
270 if(preserve)
271 RenameFile(waysx->filename_tmp,waysx->filename);
272 else
273 DeleteFile(waysx->filename_tmp);
274
275 fd=OpenFileNew(waysx->filename_tmp);
276
277 /* Sort the ways to allow separating the names */
278
279 filesort_vary(waysx->fd,fd,NULL,
280 (int (*)(const void*,const void*))sort_by_name,
281 NULL);
282
283 /* Close the files */
284
285 waysx->fd=CloseFile(waysx->fd);
286 CloseFile(fd);
287
288 /* Print the final message */
289
290 printf_last("Sorted Ways by Name: Ways=%"Pindex_t,waysx->number);
291
292
293 /* Print the start message */
294
295 printf_first("Separating Way Names: Ways=0 Names=0");
296
297 /* Re-open the file read-only and new files writeable */
298
299 waysx->fd=ReOpenFile(waysx->filename_tmp);
300
301 DeleteFile(waysx->filename_tmp);
302
303 fd=OpenFileNew(waysx->filename_tmp);
304
305 waysx->nfd=OpenFileNew(waysx->nfilename_tmp);
306
307 /* Copy from the single file into two files */
308
309 for(i=0;i<waysx->number;i++)
310 {
311 WayX wayx;
312 FILESORT_VARINT size;
313
314 ReadFile(waysx->fd,&size,FILESORT_VARSIZE);
315
316 if(namelen[nnames%2]<size)
317 names[nnames%2]=(char*)realloc((void*)names[nnames%2],namelen[nnames%2]=size);
318
319 ReadFile(waysx->fd,&wayx,sizeof(WayX));
320 ReadFile(waysx->fd,names[nnames%2],size-sizeof(WayX));
321
322 if(nnames==0 || strcmp(names[0],names[1]))
323 {
324 WriteFile(waysx->nfd,names[nnames%2],size-sizeof(WayX));
325
326 lastlength=waysx->nlength;
327 waysx->nlength+=size-sizeof(WayX);
328
329 nnames++;
330 }
331
332 wayx.way.name=lastlength;
333
334 WriteFile(fd,&wayx,sizeof(WayX));
335
336 if(!((i+1)%1000))
337 printf_middle("Separating Way Names: Ways=%"Pindex_t" Names=%"Pindex_t,i+1,nnames);
338 }
339
340 if(names[0]) free(names[0]);
341 if(names[1]) free(names[1]);
342
343 /* Close the files */
344
345 waysx->fd=CloseFile(waysx->fd);
346 CloseFile(fd);
347
348 waysx->nfd=CloseFile(waysx->nfd);
349
350 /* Print the final message */
351
352 printf_last("Separated Way Names: Ways=%"Pindex_t" Names=%"Pindex_t,waysx->number,nnames);
353
354
355 /* Print the start message */
356
357 printf_first("Sorting Ways");
358
359 /* Re-open the file read-only and a new file writeable */
360
361 waysx->fd=ReOpenFile(waysx->filename_tmp);
362
363 DeleteFile(waysx->filename_tmp);
364
365 fd=OpenFileNew(waysx->filename_tmp);
366
367 /* Allocate the array of indexes */
368
369 waysx->idata=(way_t*)malloc(waysx->number*sizeof(way_t));
370
371 assert(waysx->idata); /* Check malloc() worked */
372
373 /* Sort the ways by ID */
374
375 sortwaysx=waysx;
376
377 filesort_fixed(waysx->fd,fd,sizeof(WayX),NULL,
378 (int (*)(const void*,const void*))sort_by_id,
379 (int (*)(void*,index_t))index_by_id);
380
381 /* Close the files */
382
383 waysx->fd=CloseFile(waysx->fd);
384 CloseFile(fd);
385
386 /* Print the final message */
387
388 printf_last("Sorted Ways: Ways=%"Pindex_t,waysx->number);
389 }
390
391
392 /*++++++++++++++++++++++++++++++++++++++
393 Compact the way list, removing duplicated ways and unused ways.
394
395 WaysX *waysx The set of ways to process.
396
397 SegmentsX *segmentsx The set of segments to check.
398 ++++++++++++++++++++++++++++++++++++++*/
399
400 void CompactWayList(WaysX *waysx,SegmentsX *segmentsx)
401 {
402 int fd;
403 index_t cnumber;
404
405 /* Print the start message */
406
407 printf_first("Sorting Ways and Compacting");
408
409 /* Allocate the array of indexes */
410
411 waysx->cdata=(index_t*)malloc(waysx->number*sizeof(index_t));
412
413 assert(waysx->cdata); /* Check malloc() worked */
414
415 /* Re-open the file read-only and a new file writeable */
416
417 waysx->fd=ReOpenFile(waysx->filename_tmp);
418
419 DeleteFile(waysx->filename_tmp);
420
421 fd=OpenFileNew(waysx->filename_tmp);
422
423 /* Sort the ways to allow compacting according to the properties */
424
425 sortwaysx=waysx;
426 sortsegmentsx=segmentsx;
427
428 cnumber=filesort_fixed(waysx->fd,fd,sizeof(WayX),(int (*)(void*,index_t))delete_unused,
429 (int (*)(const void*,const void*))sort_by_name_and_prop_and_id,
430 (int (*)(void*,index_t))deduplicate_and_index_by_compact_id);
431
432 /* Close the files */
433
434 waysx->fd=CloseFile(waysx->fd);
435 CloseFile(fd);
436
437 /* Print the final message */
438
439 printf_last("Sorted and Compacted Ways: Ways=%"Pindex_t" Unique=%"Pindex_t,waysx->number,cnumber);
440 waysx->number=cnumber;
441
442 free(segmentsx->usedway);
443 segmentsx->usedway=NULL;
444 }
445
446
447 /*++++++++++++++++++++++++++++++++++++++
448 Sort the ways into id order.
449
450 int sort_by_id Returns the comparison of the id fields.
451
452 WayX *a The first extended way.
453
454 WayX *b The second extended way.
455 ++++++++++++++++++++++++++++++++++++++*/
456
457 static int sort_by_id(WayX *a,WayX *b)
458 {
459 way_t a_id=a->id;
460 way_t b_id=b->id;
461
462 if(a_id<b_id)
463 return(-1);
464 else if(a_id>b_id)
465 return(1);
466 else
467 return(FILESORT_PRESERVE_ORDER(a,b));
468 }
469
470
471 /*++++++++++++++++++++++++++++++++++++++
472 Sort the ways into name order and then id order.
473
474 int sort_by_name Returns the comparison of the name fields.
475
476 WayX *a The first extended Way.
477
478 WayX *b The second extended Way.
479 ++++++++++++++++++++++++++++++++++++++*/
480
481 static int sort_by_name(WayX *a,WayX *b)
482 {
483 int compare;
484 char *a_name=(char*)a+sizeof(WayX);
485 char *b_name=(char*)b+sizeof(WayX);
486
487 compare=strcmp(a_name,b_name);
488
489 if(compare)
490 return(compare);
491 else
492 return(FILESORT_PRESERVE_ORDER(a,b));
493 }
494
495
496 /*++++++++++++++++++++++++++++++++++++++
497 Sort the ways into name, properties and id order.
498
499 int sort_by_name_and_prop_and_id Returns the comparison of the name, properties and id fields.
500
501 WayX *a The first extended Way.
502
503 WayX *b The second extended Way.
504 ++++++++++++++++++++++++++++++++++++++*/
505
506 static int sort_by_name_and_prop_and_id(WayX *a,WayX *b)
507 {
508 int compare;
509 index_t a_name=a->way.name;
510 index_t b_name=b->way.name;
511
512 if(a_name<b_name)
513 return(-1);
514 else if(a_name>b_name)
515 return(1);
516
517 compare=WaysCompare(&a->way,&b->way);
518
519 if(compare)
520 return(compare);
521
522 return(sort_by_id(a,b));
523 }
524
525
526 /*++++++++++++++++++++++++++++++++++++++
527 Discard duplicate ways.
528
529 int deduplicate_by_id Return 1 if the value is to be kept, otherwise 0.
530
531 WayX *wayx The extended way.
532
533 index_t index The number of sorted ways that have already been written to the output file.
534 ++++++++++++++++++++++++++++++++++++++*/
535
536 static int deduplicate_by_id(WayX *wayx,index_t index)
537 {
538 static way_t previd;
539
540 if(index==0 || wayx->id!=previd)
541 {
542 previd=wayx->id;
543
544 return(1);
545 }
546 else
547 {
548 logerror("Way %"Pway_t" is duplicated.\n",wayx->id);
549
550 return(0);
551 }
552 }
553
554
555 /*++++++++++++++++++++++++++++++++++++++
556 Create the index of identifiers.
557
558 int index_by_id Return 1 if the value is to be kept, otherwise 0.
559
560 WayX *wayx The extended way.
561
562 index_t index The number of sorted ways that have already been written to the output file.
563 ++++++++++++++++++++++++++++++++++++++*/
564
565 static int index_by_id(WayX *wayx,index_t index)
566 {
567 sortwaysx->idata[index]=wayx->id;
568
569 return(1);
570 }
571
572
573 /*++++++++++++++++++++++++++++++++++++++
574 Delete the ways that are no longer being used.
575
576 int delete_unused Return 1 if the value is to be kept, otherwise 0.
577
578 WayX *wayx The extended way.
579
580 index_t index The number of unsorted ways that have been read from the input file.
581 ++++++++++++++++++++++++++++++++++++++*/
582
583 static int delete_unused(WayX *wayx,index_t index)
584 {
585 if(sortsegmentsx && !IsBitSet(sortsegmentsx->usedway,index))
586 {
587 sortwaysx->cdata[index]=NO_WAY;
588
589 return(0);
590 }
591 else
592 {
593 wayx->id=index;
594
595 return(1);
596 }
597 }
598
599
600 /*++++++++++++++++++++++++++++++++++++++
601 Create the index of compacted Way identifiers and ignore Ways with duplicated properties.
602
603 int deduplicate_and_index_by_compact_id Return 1 if the value is to be kept, otherwise 0.
604
605 WayX *wayx The extended way.
606
607 index_t index The number of sorted ways that have already been written to the output file.
608 ++++++++++++++++++++++++++++++++++++++*/
609
610 static int deduplicate_and_index_by_compact_id(WayX *wayx,index_t index)
611 {
612 static Way lastway;
613
614 if(index==0 || wayx->way.name!=lastway.name || WaysCompare(&lastway,&wayx->way))
615 {
616 lastway=wayx->way;
617
618 sortwaysx->cdata[wayx->id]=index;
619
620 wayx->id=index;
621
622 return(1);
623 }
624 else
625 {
626 sortwaysx->cdata[wayx->id]=index-1;
627
628 return(0);
629 }
630 }
631
632
633 /*++++++++++++++++++++++++++++++++++++++
634 Find a particular way index.
635
636 index_t IndexWayX Returns the index of the extended way with the specified id.
637
638 WaysX *waysx The set of ways to process.
639
640 way_t id The way id to look for.
641 ++++++++++++++++++++++++++++++++++++++*/
642
643 index_t IndexWayX(WaysX *waysx,way_t id)
644 {
645 index_t start=0;
646 index_t end=waysx->number-1;
647 index_t mid;
648
649 /* Binary search - search key exact match only is required.
650 *
651 * # <- start | Check mid and move start or end if it doesn't match
652 * # |
653 * # | Since an exact match is wanted we can set end=mid-1
654 * # <- mid | or start=mid+1 because we know that mid doesn't match.
655 * # |
656 * # | Eventually either end=start or end=start+1 and one of
657 * # <- end | start or end is the wanted one.
658 */
659
660 if(end<start) /* There are no ways */
661 return(NO_WAY);
662 else if(id<waysx->idata[start]) /* Check key is not before start */
663 return(NO_WAY);
664 else if(id>waysx->idata[end]) /* Check key is not after end */
665 return(NO_WAY);
666 else
667 {
668 do
669 {
670 mid=(start+end)/2; /* Choose mid point */
671
672 if(waysx->idata[mid]<id) /* Mid point is too low */
673 start=mid+1;
674 else if(waysx->idata[mid]>id) /* Mid point is too high */
675 end=mid?(mid-1):mid;
676 else /* Mid point is correct */
677 return(mid);
678 }
679 while((end-start)>1);
680
681 if(waysx->idata[start]==id) /* Start is correct */
682 return(start);
683
684 if(waysx->idata[end]==id) /* End is correct */
685 return(end);
686 }
687
688 return(NO_WAY);
689 }
690
691
692 /*++++++++++++++++++++++++++++++++++++++
693 Save the way list to a file.
694
695 WaysX *waysx The set of ways to save.
696
697 const char *filename The name of the file to save.
698 ++++++++++++++++++++++++++++++++++++++*/
699
700 void SaveWayList(WaysX *waysx,const char *filename)
701 {
702 index_t i;
703 int fd;
704 int position=0;
705 WayX wayx;
706 WaysFile waysfile={0};
707 highways_t highways=0;
708 transports_t allow=0;
709 properties_t props=0;
710
711 /* Print the start message */
712
713 printf_first("Writing Ways: Ways=0");
714
715 /* Re-open the files */
716
717 waysx->fd=ReOpenFile(waysx->filename_tmp);
718 waysx->nfd=ReOpenFile(waysx->nfilename_tmp);
719
720 /* Write out the ways data */
721
722 fd=OpenFileNew(filename);
723
724 SeekFile(fd,sizeof(WaysFile));
725
726 for(i=0;i<waysx->number;i++)
727 {
728 ReadFile(waysx->fd,&wayx,sizeof(WayX));
729
730 highways|=HIGHWAYS(wayx.way.type);
731 allow |=wayx.way.allow;
732 props |=wayx.way.props;
733
734 WriteFile(fd,&wayx.way,sizeof(Way));
735
736 if(!((i+1)%1000))
737 printf_middle("Writing Ways: Ways=%"Pindex_t,i+1);
738 }
739
740 /* Write out the ways names */
741
742 SeekFile(fd,sizeof(WaysFile)+(off_t)waysx->number*sizeof(Way));
743
744 while(position<waysx->nlength)
745 {
746 int len=1024;
747 char temp[1024];
748
749 if((waysx->nlength-position)<1024)
750 len=waysx->nlength-position;
751
752 ReadFile(waysx->nfd,temp,len);
753
754 WriteFile(fd,temp,len);
755
756 position+=len;
757 }
758
759 /* Close the files */
760
761 waysx->fd=CloseFile(waysx->fd);
762 waysx->nfd=CloseFile(waysx->nfd);
763
764 /* Write out the header structure */
765
766 waysfile.number =waysx->number;
767
768 waysfile.highways=highways;
769 waysfile.allow =allow;
770 waysfile.props =props;
771
772 SeekFile(fd,0);
773 WriteFile(fd,&waysfile,sizeof(WaysFile));
774
775 CloseFile(fd);
776
777 /* Print the final message */
778
779 printf_last("Wrote Ways: Ways=%"Pindex_t,waysx->number);
780 }

Properties

Name Value
cvs:description Extended ways functions.