Routino SVN Repository Browser

Check out the latest version of Routino: svn co http://routino.org/svn/trunk routino

ViewVC logotype

Contents of /trunk/src/errorlogx.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1408 - (show annotations) (download) (as text)
Fri Jun 21 14:43:37 2013 UTC (11 years, 8 months ago) by amb
File MIME type: text/x-csrc
File size: 25300 byte(s)
Use the new functions for buffering while reading when looping through files
other than the ones already done that use the FILESORT_VARINT method of storing
data.

1 /***************************************
2 Error log processing functions.
3
4 Part of the Routino routing software.
5 ******************/ /******************
6 This file Copyright 2013 Andrew M. Bishop
7
8 This program is free software: you can redistribute it and/or modify
9 it under the terms of the GNU Affero General Public License as published by
10 the Free Software Foundation, either version 3 of the License, or
11 (at your option) any later version.
12
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU Affero General Public License for more details.
17
18 You should have received a copy of the GNU Affero General Public License
19 along with this program. If not, see <http://www.gnu.org/licenses/>.
20 ***************************************/
21
22
23 #include "typesx.h"
24 #include "nodesx.h"
25 #include "waysx.h"
26 #include "relationsx.h"
27
28 #include "errorlogx.h"
29 #include "errorlog.h"
30
31 #include "files.h"
32 #include "sorting.h"
33
34
35 /* Global variables */
36
37 /*+ The name of the error log file. +*/
38 extern char *errorlogfilename;
39
40 /*+ The name of the binary error log file. +*/
41 extern char *errorbinfilename;
42
43 /* Local variables */
44
45 /*+ Temporary file-local variables for use by the sort functions. +*/
46 static latlong_t lat_min,lat_max,lon_min,lon_max;
47
48 /* Local functions */
49
50 static void reindex_nodes(NodesX *nodesx);
51 static void reindex_ways(WaysX *waysx);
52 static void reindex_relations(RelationsX *relationsx);
53
54 static int lookup_lat_long_node(NodesX *nodesx,node_t node,latlong_t *latitude,latlong_t *longitude);
55 static int lookup_lat_long_way(WaysX *waysx,NodesX *nodesx,way_t way,latlong_t *latitude,latlong_t *longitude,index_t error);
56 static int lookup_lat_long_relation(RelationsX *relationsx,WaysX *waysx,NodesX *nodesx,relation_t relation,latlong_t *latitude,latlong_t *longitude,index_t error);
57
58 static int sort_by_lat_long(ErrorLogX *a,ErrorLogX *b);
59 static int measure_lat_long(ErrorLogX *errorlogx,index_t index);
60
61
62 /*++++++++++++++++++++++++++++++++++++++
63 Allocate a new error log list (create a new file).
64
65 ErrorLogsX *NewErrorLogList Returns a pointer to the error log list.
66 ++++++++++++++++++++++++++++++++++++++*/
67
68 ErrorLogsX *NewErrorLogList(void)
69 {
70 ErrorLogsX *errorlogsx;
71
72 errorlogsx=(ErrorLogsX*)calloc(1,sizeof(ErrorLogsX));
73
74 logassert(errorlogsx,"Failed to allocate memory (try using slim mode?)"); /* Check calloc() worked */
75
76 return(errorlogsx);
77 }
78
79
80 /*++++++++++++++++++++++++++++++++++++++
81 Free an error log list.
82
83 ErrorLogsX *errorlogsx The set of error logs to be freed.
84 ++++++++++++++++++++++++++++++++++++++*/
85
86 void FreeErrorLogList(ErrorLogsX *errorlogsx)
87 {
88 free(errorlogsx);
89 }
90
91
92 /*++++++++++++++++++++++++++++++++++++++
93 Process the binary error log.
94
95 ErrorLogsX *errorlogsx The set of error logs to update.
96
97 NodesX *nodesx The set of nodes.
98
99 WaysX *waysx The set of ways.
100
101 RelationsX *relationsx The set of relations.
102 ++++++++++++++++++++++++++++++++++++++*/
103
104 void ProcessErrorLogs(ErrorLogsX *errorlogsx,NodesX *nodesx,WaysX *waysx,RelationsX *relationsx)
105 {
106 int oldfd,newfd;
107 uint32_t offset=0;
108 int nerrorlogobjects=0;
109 int finished;
110 ErrorLogObject errorlogobjects[8];
111
112 /* Re-index the nodes, ways and relations */
113
114 printf_first("Re-indexing the Data: Nodes=0 Ways=0 Route-Relations=0 Turn-Relations=0");
115
116 reindex_nodes(nodesx);
117
118 printf_middle("Re-indexing the Data: Nodes=%"Pindex_t" Ways=0 Route-Relations=0 Turn-Relations=0",nodesx->number);
119
120 reindex_ways(waysx);
121
122 printf_middle("Re-indexing the Data: Nodes=%"Pindex_t" Ways=%"Pindex_t" Route-Relations=0 Turn-Relations=0",nodesx->number,waysx->number);
123
124 reindex_relations(relationsx);
125
126 printf_last("Re-indexed the Data: Nodes=%"Pindex_t" Ways=%"Pindex_t" Route-Relations=%"Pindex_t" Turn-Relations=%"Pindex_t,nodesx->number,waysx->number,relationsx->rrnumber,relationsx->trnumber);
127
128
129 /* Print the start message */
130
131 printf_first("Calculating Coordinates: Errors=0");
132
133 /* Map into memory / open the files */
134
135 #if !SLIM
136 nodesx->data=MapFile(nodesx->filename);
137 #else
138 nodesx->fd=ReOpenFile(nodesx->filename);
139
140 InvalidateNodeXCache(nodesx->cache);
141 #endif
142
143 waysx->fd=ReOpenFile(waysx->filename);
144 relationsx->rrfd=ReOpenFile(relationsx->rrfilename);
145 relationsx->trfd=ReOpenFile(relationsx->trfilename);
146
147 /* Open the binary log file read-only and a new file writeable */
148
149 oldfd=ReOpenFileBuffered(errorbinfilename);
150
151 DeleteFile(errorbinfilename);
152
153 newfd=OpenFileBufferedNew(errorbinfilename);
154
155 /* Loop through the file and merge the raw data into coordinates */
156
157 errorlogsx->number=0;
158
159 do
160 {
161 ErrorLogObject errorlogobject;
162
163 finished=ReadFileBuffered(oldfd,&errorlogobject,sizeof(ErrorLogObject));
164
165 if(finished)
166 errorlogobject.offset=SizeFile(errorlogfilename);
167
168 if(offset!=errorlogobject.offset)
169 {
170 ErrorLogX errorlogx;
171 latlong_t errorlat=NO_LATLONG,errorlon=NO_LATLONG;
172
173 /* Calculate suitable coordinates */
174
175 if(nerrorlogobjects==1)
176 {
177 if(errorlogobjects[0].type=='N')
178 {
179 node_t node=(node_t)errorlogobjects[0].id;
180
181 lookup_lat_long_node(nodesx,node,&errorlat,&errorlon);
182 }
183 else if(errorlogobjects[0].type=='W')
184 {
185 way_t way=(way_t)errorlogobjects[0].id;
186
187 lookup_lat_long_way(waysx,nodesx,way,&errorlat,&errorlon,errorlogsx->number);
188 }
189 else if(errorlogobjects[0].type=='R')
190 {
191 relation_t relation=(relation_t)errorlogobjects[0].type;
192
193 lookup_lat_long_relation(relationsx,waysx,nodesx,relation,&errorlat,&errorlon,errorlogsx->number);
194 }
195 }
196 else
197 {
198 latlong_t latitude[8],longitude[8];
199 int i;
200 int ncoords=0,nnodes=0,nways=0,nrelations=0;
201
202 for(i=0;i<nerrorlogobjects;i++)
203 {
204 if(errorlogobjects[i].type=='N')
205 {
206 node_t node=(node_t)errorlogobjects[i].id;
207
208 if(lookup_lat_long_node(nodesx,node,&latitude[ncoords],&longitude[ncoords]))
209 ncoords++;
210
211 nnodes++;
212 }
213 else if(errorlogobjects[i].type=='W')
214 nways++;
215 else if(errorlogobjects[i].type=='R')
216 nrelations++;
217 }
218
219 if(nways==0 && nrelations==0) /* only nodes */
220 ;
221 else if(ncoords) /* some good nodes, possibly ways and/or relations */
222 ;
223 else if(nways) /* no good nodes, possibly some good ways */
224 {
225 for(i=0;i<nerrorlogobjects;i++)
226 if(errorlogobjects[i].type=='W')
227 {
228 way_t way=(way_t)errorlogobjects[i].id;
229
230 if(lookup_lat_long_way(waysx,nodesx,way,&latitude[ncoords],&longitude[ncoords],errorlogsx->number))
231 ncoords++;
232 }
233 }
234
235 if(nrelations==0) /* only nodes and/or ways */
236 ;
237 else if(ncoords) /* some good nodes and/or ways, possibly relations */
238 ;
239 else /* if(nrelations) */
240 {
241 for(i=0;i<nerrorlogobjects;i++)
242 if(errorlogobjects[i].type=='R')
243 {
244 relation_t relation=(relation_t)errorlogobjects[i].id;
245
246 if(lookup_lat_long_relation(relationsx,waysx,nodesx,relation,&latitude[ncoords],&longitude[ncoords],errorlogsx->number))
247 ncoords++;
248 }
249 }
250
251 if(ncoords)
252 {
253 errorlat=0;
254 errorlon=0;
255
256 for(i=0;i<ncoords;i++)
257 {
258 errorlat+=latitude[i];
259 errorlon+=longitude[i];
260 }
261
262 errorlat/=ncoords;
263 errorlon/=ncoords;
264 }
265 else
266 {
267 errorlat=NO_LATLONG;
268 errorlon=NO_LATLONG;
269 }
270 }
271
272 /* Write to file */
273
274 errorlogx.offset=offset;
275 errorlogx.length=errorlogobject.offset-offset;
276
277 errorlogx.latitude =errorlat;
278 errorlogx.longitude=errorlon;
279
280 WriteFileBuffered(newfd,&errorlogx,sizeof(ErrorLogX));
281
282 errorlogsx->number++;
283
284 offset=errorlogobject.offset;
285 nerrorlogobjects=0;
286
287 if(!(errorlogsx->number%10000))
288 printf_middle("Calculating Coordinates: Errors=%"Pindex_t,errorlogsx->number);
289 }
290
291 /* Store for later */
292
293 logassert(nerrorlogobjects<8,"Too many error log objects for one error message."); /* Only a limited amount of information stored. */
294
295 errorlogobjects[nerrorlogobjects]=errorlogobject;
296
297 nerrorlogobjects++;
298 }
299 while(!finished);
300
301 /* Unmap from memory / close the files */
302
303 #if !SLIM
304 nodesx->data=UnmapFile(nodesx->data);
305 #else
306 nodesx->fd=CloseFile(nodesx->fd);
307 #endif
308
309 waysx->fd=CloseFile(waysx->fd);
310 relationsx->rrfd=CloseFile(relationsx->rrfd);
311 relationsx->trfd=CloseFile(relationsx->trfd);
312
313 CloseFileBuffered(oldfd);
314 CloseFileBuffered(newfd);
315
316 /* Print the final message */
317
318 printf_last("Calculated Coordinates: Errors=%"Pindex_t,errorlogsx->number);
319 }
320
321
322 /*++++++++++++++++++++++++++++++++++++++
323 Re-index the nodes that were kept.
324
325 NodesX *nodesx The set of nodes to process (contains the filename and number of nodes).
326 ++++++++++++++++++++++++++++++++++++++*/
327
328 static void reindex_nodes(NodesX *nodesx)
329 {
330 int fd;
331 index_t index=0;
332 NodeX nodex;
333
334 nodesx->number=nodesx->knumber;
335
336 nodesx->idata=(node_t*)malloc(nodesx->number*sizeof(node_t));
337
338 /* Get the node id for each node in the file. */
339
340 fd=ReOpenFileBuffered(nodesx->filename);
341
342 while(!ReadFileBuffered(fd,&nodex,sizeof(NodeX)))
343 {
344 nodesx->idata[index]=nodex.id;
345
346 index++;
347 }
348
349 CloseFileBuffered(fd);
350 }
351
352
353 /*++++++++++++++++++++++++++++++++++++++
354 Re-index the ways that were kept.
355
356 WaysX *waysx The set of ways to process (contains the filename and number of ways).
357 ++++++++++++++++++++++++++++++++++++++*/
358
359 static void reindex_ways(WaysX *waysx)
360 {
361 FILESORT_VARINT waysize;
362 int fd;
363 off_t position=0;
364 index_t index=0;
365
366 waysx->number=waysx->knumber;
367
368 waysx->idata=(way_t*)malloc(waysx->number*sizeof(way_t));
369 waysx->odata=(off_t*)malloc(waysx->number*sizeof(off_t));
370
371 /* Get the way id and the offset for each way in the file */
372
373 fd=ReOpenFileBuffered(waysx->filename);
374
375 while(!ReadFileBuffered(fd,&waysize,FILESORT_VARSIZE))
376 {
377 WayX wayx;
378
379 ReadFileBuffered(fd,&wayx,sizeof(WayX));
380
381 waysx->idata[index]=wayx.id;
382 waysx->odata[index]=position+FILESORT_VARSIZE+sizeof(WayX);
383
384 index++;
385
386 SkipFileBuffered(fd,waysize-sizeof(WayX));
387
388 position+=waysize+FILESORT_VARSIZE;
389 }
390
391 CloseFileBuffered(fd);
392 }
393
394
395 /*++++++++++++++++++++++++++++++++++++++
396 Re-index the relations that were kept.
397
398 RelationsX *relationsx The set of relations to process (contains the filenames and numbers of relations).
399 ++++++++++++++++++++++++++++++++++++++*/
400
401 static void reindex_relations(RelationsX *relationsx)
402 {
403 FILESORT_VARINT relationsize;
404 int fd;
405 off_t position=0;
406 index_t index;
407 TurnRelX turnrelx;
408
409 /* Route relations */
410
411 relationsx->rrnumber=relationsx->rrknumber;
412
413 relationsx->rridata=(relation_t*)malloc(relationsx->rrnumber*sizeof(relation_t));
414 relationsx->rrodata=(off_t*)malloc(relationsx->rrnumber*sizeof(off_t));
415
416 /* Get the relation id and the offset for each relation in the file */
417
418 fd=ReOpenFileBuffered(relationsx->rrfilename);
419
420 index=0;
421
422 while(!ReadFileBuffered(fd,&relationsize,FILESORT_VARSIZE))
423 {
424 RouteRelX routerelx;
425
426 ReadFileBuffered(fd,&routerelx,sizeof(RouteRelX));
427
428 relationsx->rridata[index]=routerelx.id;
429 relationsx->rrodata[index]=position+FILESORT_VARSIZE+sizeof(RouteRelX);
430
431 index++;
432
433 SkipFileBuffered(fd,relationsize-sizeof(RouteRelX));
434
435 position+=relationsize+FILESORT_VARSIZE;
436 }
437
438 CloseFileBuffered(fd);
439
440
441 /* Turn relations */
442
443 relationsx->trnumber=relationsx->trknumber;
444
445 relationsx->tridata=(relation_t*)malloc(relationsx->trnumber*sizeof(relation_t));
446
447 /* Get the relation id and the offset for each relation in the file */
448
449 fd=ReOpenFileBuffered(relationsx->trfilename);
450
451 index=0;
452
453 while(!ReadFileBuffered(fd,&turnrelx,sizeof(TurnRelX)))
454 {
455 relationsx->tridata[index]=turnrelx.id;
456
457 index++;
458 }
459
460 CloseFileBuffered(fd);
461 }
462
463
464 /*++++++++++++++++++++++++++++++++++++++
465 Lookup a node's latitude and longitude.
466
467 int lookup_lat_long_node Returns 1 if a node was found.
468
469 NodesX *nodesx The set of nodes to use.
470
471 node_t node The node number.
472
473 latlong_t *latitude Returns the latitude.
474
475 latlong_t *longitude Returns the longitude.
476 ++++++++++++++++++++++++++++++++++++++*/
477
478 static int lookup_lat_long_node(NodesX *nodesx,node_t node,latlong_t *latitude,latlong_t *longitude)
479 {
480 index_t index=IndexNodeX(nodesx,node);
481
482 if(index==NO_NODE)
483 return 0;
484 else
485 {
486 NodeX *nodex=LookupNodeX(nodesx,index,1);
487
488 *latitude =nodex->latitude;
489 *longitude=nodex->longitude;
490
491 return 1;
492 }
493 }
494
495
496 /*++++++++++++++++++++++++++++++++++++++
497 Lookup a way's latitude and longitude.
498
499 int lookup_lat_long_way Returns 1 if a way was found.
500
501 WaysX *waysx The set of ways to use.
502
503 NodesX *nodesx The set of nodes to use.
504
505 way_t way The way number.
506
507 latlong_t *latitude Returns the latitude.
508
509 latlong_t *longitude Returns the longitude.
510
511 index_t error The index of the error in the complete set of errors.
512 ++++++++++++++++++++++++++++++++++++++*/
513
514 static int lookup_lat_long_way(WaysX *waysx,NodesX *nodesx,way_t way,latlong_t *latitude,latlong_t *longitude,index_t error)
515 {
516 index_t index=IndexWayX(waysx,way);
517
518 if(index==NO_WAY)
519 return 0;
520 else
521 {
522 int count=1;
523 off_t offset=waysx->odata[index];
524 node_t node1,node2,prevnode,node;
525 latlong_t latitude1,longitude1,latitude2,longitude2;
526
527 SeekFile(waysx->fd,offset);
528
529 /* Choose a random pair of adjacent nodes */
530
531 if(ReadFile(waysx->fd,&node1,sizeof(node_t)) || node1==NO_NODE_ID)
532 return 0;
533
534 if(ReadFile(waysx->fd,&node2,sizeof(node_t)) || node2==NO_NODE_ID)
535 return lookup_lat_long_node(nodesx,node1,latitude,longitude);
536
537 prevnode=node2;
538
539 while(!ReadFile(waysx->fd,&node,sizeof(node_t)) && node!=NO_NODE_ID)
540 {
541 count++;
542
543 if((error%count)==0) /* A 1/count chance */
544 {
545 node1=prevnode;
546 node2=node;
547 }
548
549 prevnode=node;
550 }
551
552 if(!lookup_lat_long_node(nodesx,node1,&latitude1,&longitude1))
553 return lookup_lat_long_node(nodesx,node2,latitude,longitude);
554
555 if(!lookup_lat_long_node(nodesx,node2,&latitude2,&longitude2))
556 return lookup_lat_long_node(nodesx,node1,latitude,longitude);
557
558 *latitude =(latitude1 +latitude2 )/2;
559 *longitude=(longitude1+longitude2)/2;
560
561 return 1;
562 }
563 }
564
565
566 /*++++++++++++++++++++++++++++++++++++++
567 Lookup a relation's latitude and longitude.
568
569 int lookup_lat_long_relation Returns 1 if a relation was found.
570
571 RelationsX *relationsx The set of relations to use.
572
573 WaysX *waysx The set of ways to use.
574
575 NodesX *nodesx The set of nodes to use.
576
577 relation_t relation The relation number.
578
579 latlong_t *latitude Returns the latitude.
580
581 latlong_t *longitude Returns the longitude.
582
583 index_t error The index of the error in the complete set of errors.
584 ++++++++++++++++++++++++++++++++++++++*/
585
586 static int lookup_lat_long_relation(RelationsX *relationsx,WaysX *waysx,NodesX *nodesx,relation_t relation,latlong_t *latitude,latlong_t *longitude,index_t error)
587 {
588 index_t index=IndexRouteRelX(relationsx,relation);
589
590 if(index==NO_RELATION)
591 {
592 index=IndexTurnRelX(relationsx,relation);
593
594 if(index==NO_RELATION)
595 return 0;
596 else
597 {
598 TurnRelX turnrelx;
599
600 SeekReadFile(relationsx->trfd,&turnrelx,sizeof(TurnRelX),index*sizeof(TurnRelX));
601
602 if(lookup_lat_long_node(nodesx,turnrelx.via,latitude,longitude))
603 return 1;
604
605 if(lookup_lat_long_way(waysx,nodesx,turnrelx.from,latitude,longitude,error))
606 return 1;
607
608 if(lookup_lat_long_way(waysx,nodesx,turnrelx.to,latitude,longitude,error))
609 return 1;
610
611 return 0;
612 }
613 }
614 else
615 {
616 int count;
617 off_t offset=relationsx->rrodata[index];
618 node_t node=NO_NODE_ID,tempnode;
619 way_t way=NO_WAY_ID,tempway;
620 relation_t relation=NO_RELATION_ID,temprelation;
621
622 SeekFile(relationsx->rrfd,offset);
623
624 /* Choose a random node */
625
626 count=0;
627
628 while(!ReadFile(relationsx->rrfd,&tempnode,sizeof(node_t)) && tempnode!=NO_NODE_ID)
629 {
630 count++;
631
632 if((error%count)==0) /* A 1/count chance */
633 node=tempnode;
634 }
635
636 if(lookup_lat_long_node(nodesx,node,latitude,longitude))
637 return 1;
638
639 /* Choose a random way */
640
641 count=0;
642
643 while(!ReadFile(relationsx->rrfd,&tempway,sizeof(way_t)) && tempway!=NO_WAY_ID)
644 {
645 count++;
646
647 if((error%count)==0) /* A 1/count chance */
648 way=tempway;
649 }
650
651 if(lookup_lat_long_way(waysx,nodesx,way,latitude,longitude,error))
652 return 1;
653
654 /* Choose a random relation */
655
656 count=0;
657
658 while(!ReadFile(relationsx->rrfd,&temprelation,sizeof(relation_t)) && temprelation!=NO_RELATION_ID)
659 {
660 count++;
661
662 if((error%count)==0) /* A 1/count chance */
663 relation=temprelation;
664 }
665
666 return lookup_lat_long_relation(relationsx,waysx,nodesx,relation,latitude,longitude,error);
667 }
668 }
669
670
671 /*++++++++++++++++++++++++++++++++++++++
672 Sort the error logs geographically.
673
674 ErrorLogsX *errorlogsx The set of error logs to sort.
675 ++++++++++++++++++++++++++++++++++++++*/
676
677 void SortErrorLogsGeographically(ErrorLogsX *errorlogsx)
678 {
679 int oldfd,newfd;
680 ll_bin_t lat_min_bin,lat_max_bin,lon_min_bin,lon_max_bin;
681
682 /* Print the start message */
683
684 printf_first("Sorting Errors Geographically");
685
686 /* Work out the range of data */
687
688 lat_min=radians_to_latlong( 2);
689 lat_max=radians_to_latlong(-2);
690 lon_min=radians_to_latlong( 4);
691 lon_max=radians_to_latlong(-4);
692
693 /* Re-open the file read-only and a new file writeable */
694
695 oldfd=ReOpenFile(errorbinfilename);
696
697 DeleteFile(errorbinfilename);
698
699 newfd=OpenFileNew(errorbinfilename);
700
701 /* Sort errors geographically */
702
703 filesort_fixed(oldfd,newfd,sizeof(ErrorLogX),NULL,
704 (int (*)(const void*,const void*))sort_by_lat_long,
705 (int (*)(void*,index_t))measure_lat_long);
706
707 /* Close the files */
708
709 CloseFile(oldfd);
710 CloseFile(newfd);
711
712 /* Work out the number of bins */
713
714 lat_min_bin=latlong_to_bin(lat_min);
715 lon_min_bin=latlong_to_bin(lon_min);
716 lat_max_bin=latlong_to_bin(lat_max);
717 lon_max_bin=latlong_to_bin(lon_max);
718
719 errorlogsx->latzero=lat_min_bin;
720 errorlogsx->lonzero=lon_min_bin;
721
722 errorlogsx->latbins=(lat_max_bin-lat_min_bin)+1;
723 errorlogsx->lonbins=(lon_max_bin-lon_min_bin)+1;
724
725 /* Print the final message */
726
727 printf_last("Sorted Errors Geographically: Errors=%"Pindex_t,errorlogsx->number);
728 }
729
730
731 /*++++++++++++++++++++++++++++++++++++++
732 Sort the errors into latitude and longitude order (first by longitude bin
733 number, then by latitude bin number and then by exact longitude and then by
734 exact latitude).
735
736 int sort_by_lat_long Returns the comparison of the latitude and longitude fields.
737
738 ErrorLogX *a The first error location.
739
740 ErrorLogX *b The second error location.
741 ++++++++++++++++++++++++++++++++++++++*/
742
743 static int sort_by_lat_long(ErrorLogX *a,ErrorLogX *b)
744 {
745 ll_bin_t a_lon=latlong_to_bin(a->longitude);
746 ll_bin_t b_lon=latlong_to_bin(b->longitude);
747
748 if(a_lon<b_lon)
749 return(-1);
750 else if(a_lon>b_lon)
751 return(1);
752 else
753 {
754 ll_bin_t a_lat=latlong_to_bin(a->latitude);
755 ll_bin_t b_lat=latlong_to_bin(b->latitude);
756
757 if(a_lat<b_lat)
758 return(-1);
759 else if(a_lat>b_lat)
760 return(1);
761 else
762 {
763 if(a->longitude<b->longitude)
764 return(-1);
765 else if(a->longitude>b->longitude)
766 return(1);
767 else
768 {
769 if(a->latitude<b->latitude)
770 return(-1);
771 else if(a->latitude>b->latitude)
772 return(1);
773 }
774
775 return(FILESORT_PRESERVE_ORDER(a,b));
776 }
777 }
778 }
779
780
781 /*++++++++++++++++++++++++++++++++++++++
782 Measure the extent of the data.
783
784 int measure_lat_long Return 1 if the value is to be kept, otherwise 0.
785
786 ErrorLogX *errorlogx The error location.
787
788 index_t index The number of sorted error locations that have already been written to the output file.
789 ++++++++++++++++++++++++++++++++++++++*/
790
791 static int measure_lat_long(ErrorLogX *errorlogx,index_t index)
792 {
793 if(errorlogx->latitude!=NO_LATLONG)
794 {
795 if(errorlogx->latitude<lat_min)
796 lat_min=errorlogx->latitude;
797 if(errorlogx->latitude>lat_max)
798 lat_max=errorlogx->latitude;
799 if(errorlogx->longitude<lon_min)
800 lon_min=errorlogx->longitude;
801 if(errorlogx->longitude>lon_max)
802 lon_max=errorlogx->longitude;
803 }
804
805 return(1);
806 }
807
808
809 /*++++++++++++++++++++++++++++++++++++++
810 Save the binary error log.
811
812 ErrorLogsX *errorlogsx The set of error logs to write.
813
814 char *filename The name of the final file to write.
815 ++++++++++++++++++++++++++++++++++++++*/
816
817 void SaveErrorLogs(ErrorLogsX *errorlogsx,char *filename)
818 {
819 ErrorLogsFile errorlogsfile;
820 ErrorLogX errorlogx;
821 int oldfd,newfd;
822 ll_bin2_t latlonbin=0,maxlatlonbins;
823 index_t *offsets;
824 index_t number=0,number_geo=0,number_nongeo=0;
825 off_t size;
826
827 /* Print the start message */
828
829 printf_first("Writing Errors: Geographical=0 Non-geographical=0");
830
831 /* Allocate the memory for the geographical offsets array */
832
833 offsets=(index_t*)malloc((errorlogsx->latbins*errorlogsx->lonbins+1)*sizeof(index_t));
834
835 logassert(offsets,"Failed to allocate memory (try using slim mode?)"); /* Check malloc() worked */
836
837 latlonbin=0;
838
839 /* Re-open the file */
840
841 oldfd=ReOpenFileBuffered(errorbinfilename);
842
843 newfd=OpenFileBufferedNew(filename);
844
845 /* Write out the geographical errors */
846
847 SeekFileBuffered(newfd,sizeof(ErrorLogsFile)+(errorlogsx->latbins*errorlogsx->lonbins+1)*sizeof(index_t));
848
849 while(!ReadFileBuffered(oldfd,&errorlogx,sizeof(ErrorLogX)))
850 {
851 ErrorLog errorlog={0};
852 ll_bin_t latbin,lonbin;
853 ll_bin2_t llbin;
854
855 if(errorlogx.latitude==NO_LATLONG)
856 continue;
857
858 /* Create the ErrorLog */
859
860 errorlog.latoffset=latlong_to_off(errorlogx.latitude);
861 errorlog.lonoffset=latlong_to_off(errorlogx.longitude);
862
863 errorlog.offset=errorlogx.offset;
864 errorlog.length=errorlogx.length;
865
866 /* Work out the offsets */
867
868 latbin=latlong_to_bin(errorlogx.latitude )-errorlogsx->latzero;
869 lonbin=latlong_to_bin(errorlogx.longitude)-errorlogsx->lonzero;
870 llbin=lonbin*errorlogsx->latbins+latbin;
871
872 for(;latlonbin<=llbin;latlonbin++)
873 offsets[latlonbin]=number_geo;
874
875 /* Write the data */
876
877 WriteFileBuffered(newfd,&errorlog,sizeof(ErrorLog));
878
879 number_geo++;
880 number++;
881
882 if(!(number%10000))
883 printf_middle("Writing Errors: Geographical=%"Pindex_t" Non-geographical=%"Pindex_t,number_geo,number_nongeo);
884 }
885
886 /* Write out the non-geographical errors */
887
888 SeekFileBuffered(oldfd,0);
889
890 while(!ReadFileBuffered(oldfd,&errorlogx,sizeof(ErrorLogX)))
891 {
892 ErrorLog errorlog={0};
893
894 if(errorlogx.latitude!=NO_LATLONG)
895 continue;
896
897 /* Create the ErrorLog */
898
899 errorlog.latoffset=0;
900 errorlog.lonoffset=0;
901
902 errorlog.offset=errorlogx.offset;
903 errorlog.length=errorlogx.length;
904
905 /* Write the data */
906
907 WriteFileBuffered(newfd,&errorlog,sizeof(ErrorLog));
908
909 number_nongeo++;
910 number++;
911
912 if(!(number%10000))
913 printf_middle("Writing Errors: Geographical=%"Pindex_t" Non-geographical=%"Pindex_t,number_geo,number_nongeo);
914 }
915
916 /* Close the input file */
917
918 CloseFileBuffered(oldfd);
919
920 DeleteFile(errorbinfilename);
921
922 /* Append the text from the log file */
923
924 size=SizeFile(errorlogfilename);
925
926 oldfd=ReOpenFile(errorlogfilename);
927
928 while(size)
929 {
930 int i;
931 char buffer[4096];
932 off_t chunksize=(size>sizeof(buffer)?sizeof(buffer):size);
933
934 ReadFile(oldfd,buffer,chunksize);
935
936 for(i=0;i<chunksize;i++)
937 if(buffer[i]=='\n')
938 buffer[i]=0;
939
940 WriteFileBuffered(newfd,buffer,chunksize);
941
942 size-=chunksize;
943 }
944
945 CloseFile(oldfd);
946
947 /* Finish off the offset indexing and write them out */
948
949 maxlatlonbins=errorlogsx->latbins*errorlogsx->lonbins;
950
951 for(;latlonbin<=maxlatlonbins;latlonbin++)
952 offsets[latlonbin]=number_geo;
953
954 SeekFileBuffered(newfd,sizeof(ErrorLogsFile));
955 WriteFileBuffered(newfd,offsets,(errorlogsx->latbins*errorlogsx->lonbins+1)*sizeof(index_t));
956
957 free(offsets);
958
959 /* Write out the header structure */
960
961 errorlogsfile.number =number;
962 errorlogsfile.number_geo =number_geo;
963 errorlogsfile.number_nongeo=number_nongeo;
964
965 errorlogsfile.latbins=errorlogsx->latbins;
966 errorlogsfile.lonbins=errorlogsx->lonbins;
967
968 errorlogsfile.latzero=errorlogsx->latzero;
969 errorlogsfile.lonzero=errorlogsx->lonzero;
970
971 SeekFileBuffered(newfd,0);
972 WriteFileBuffered(newfd,&errorlogsfile,sizeof(ErrorLogsFile));
973
974 CloseFileBuffered(newfd);
975
976 /* Print the final message */
977
978 printf_last("Wrote Errors: Geographical=%"Pindex_t" Non-geographical=%"Pindex_t,number_geo,number_nongeo);
979 }