Routino SVN Repository Browser

Check out the latest version of Routino: svn co http://routino.org/svn/trunk routino

ViewVC logotype

Contents of /trunk/src/errorlogx.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1784 - (show annotations) (download) (as text)
Sat Aug 15 13:08:37 2015 UTC (9 years, 7 months ago) by amb
File MIME type: text/x-csrc
File size: 25859 byte(s)
Merge libroutino branch back into the trunk.

1 /***************************************
2 Error log processing functions.
3
4 Part of the Routino routing software.
5 ******************/ /******************
6 This file Copyright 2013-2015 Andrew M. Bishop
7
8 This program is free software: you can redistribute it and/or modify
9 it under the terms of the GNU Affero General Public License as published by
10 the Free Software Foundation, either version 3 of the License, or
11 (at your option) any later version.
12
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU Affero General Public License for more details.
17
18 You should have received a copy of the GNU Affero General Public License
19 along with this program. If not, see <http://www.gnu.org/licenses/>.
20 ***************************************/
21
22
23 #include "typesx.h"
24 #include "nodesx.h"
25 #include "waysx.h"
26 #include "relationsx.h"
27
28 #include "errorlogx.h"
29 #include "errorlog.h"
30
31 #include "files.h"
32 #include "sorting.h"
33
34
35 /* Global variables */
36
37 /*+ The name of the error log file. +*/
38 extern char *errorlogfilename;
39
40 /*+ The name of the binary error log file. +*/
41 extern char *errorbinfilename;
42
43 /* Local variables */
44
45 /*+ Temporary file-local variables for use by the sort functions (re-initialised for each sort). +*/
46 static latlong_t lat_min,lat_max,lon_min,lon_max;
47
48 /* Local functions */
49
50 static void reindex_nodes(NodesX *nodesx);
51 static void reindex_ways(WaysX *waysx);
52 static void reindex_relations(RelationsX *relationsx);
53
54 static int lookup_lat_long_node(NodesX *nodesx,node_t node,latlong_t *latitude,latlong_t *longitude);
55 static int lookup_lat_long_way(WaysX *waysx,NodesX *nodesx,way_t way,latlong_t *latitude,latlong_t *longitude,index_t error);
56 static int lookup_lat_long_relation(RelationsX *relationsx,WaysX *waysx,NodesX *nodesx,relation_t relation,latlong_t *latitude,latlong_t *longitude,index_t error);
57
58 static int sort_by_lat_long(ErrorLogX *a,ErrorLogX *b);
59 static int measure_lat_long(ErrorLogX *errorlogx,index_t index);
60
61
62 /*++++++++++++++++++++++++++++++++++++++
63 Allocate a new error log list (create a new file).
64
65 ErrorLogsX *NewErrorLogList Returns a pointer to the error log list.
66 ++++++++++++++++++++++++++++++++++++++*/
67
68 ErrorLogsX *NewErrorLogList(void)
69 {
70 ErrorLogsX *errorlogsx;
71
72 errorlogsx=(ErrorLogsX*)calloc(1,sizeof(ErrorLogsX));
73
74 logassert(errorlogsx,"Failed to allocate memory (try using slim mode?)"); /* Check calloc() worked */
75
76 return(errorlogsx);
77 }
78
79
80 /*++++++++++++++++++++++++++++++++++++++
81 Free an error log list.
82
83 ErrorLogsX *errorlogsx The set of error logs to be freed.
84 ++++++++++++++++++++++++++++++++++++++*/
85
86 void FreeErrorLogList(ErrorLogsX *errorlogsx)
87 {
88 free(errorlogsx);
89 }
90
91
92 /*++++++++++++++++++++++++++++++++++++++
93 Process the binary error log.
94
95 ErrorLogsX *errorlogsx The set of error logs to update.
96
97 NodesX *nodesx The set of nodes.
98
99 WaysX *waysx The set of ways.
100
101 RelationsX *relationsx The set of relations.
102 ++++++++++++++++++++++++++++++++++++++*/
103
104 void ProcessErrorLogs(ErrorLogsX *errorlogsx,NodesX *nodesx,WaysX *waysx,RelationsX *relationsx)
105 {
106 int oldfd,newfd;
107 uint32_t offset=0;
108 int nerrorlogobjects=0;
109 int finished;
110 ErrorLogObject errorlogobjects[8];
111
112 /* Re-index the nodes, ways and relations */
113
114 printf_first("Re-indexing the Data: Nodes=0 Ways=0 Route-Relations=0 Turn-Relations=0");
115
116 reindex_nodes(nodesx);
117
118 printf_middle("Re-indexing the Data: Nodes=%"Pindex_t" Ways=0 Route-Relations=0 Turn-Relations=0",nodesx->number);
119
120 reindex_ways(waysx);
121
122 printf_middle("Re-indexing the Data: Nodes=%"Pindex_t" Ways=%"Pindex_t" Route-Relations=0 Turn-Relations=0",nodesx->number,waysx->number);
123
124 reindex_relations(relationsx);
125
126 printf_last("Re-indexed the Data: Nodes=%"Pindex_t" Ways=%"Pindex_t" Route-Relations=%"Pindex_t" Turn-Relations=%"Pindex_t,nodesx->number,waysx->number,relationsx->rrnumber,relationsx->trnumber);
127
128
129 /* Print the start message */
130
131 printf_first("Calculating Coordinates: Errors=0");
132
133 /* Map into memory / open the files */
134
135 #if !SLIM
136 nodesx->data=MapFile(nodesx->filename);
137 #else
138 nodesx->fd=SlimMapFile(nodesx->filename);
139
140 InvalidateNodeXCache(nodesx->cache);
141 #endif
142
143 waysx->fd=ReOpenFileBuffered(waysx->filename);
144 relationsx->rrfd=ReOpenFileBuffered(relationsx->rrfilename);
145 relationsx->trfd=ReOpenFileBuffered(relationsx->trfilename);
146
147 /* Open the binary log file read-only and a new file writeable */
148
149 newfd=ReplaceFileBuffered(errorbinfilename,&oldfd);
150
151 /* Loop through the file and merge the raw data into coordinates */
152
153 errorlogsx->number=0;
154
155 do
156 {
157 ErrorLogObject errorlogobject;
158
159 finished=ReadFileBuffered(oldfd,&errorlogobject,sizeof(ErrorLogObject));
160
161 if(finished)
162 errorlogobject.offset=SizeFile(errorlogfilename);
163
164 if(offset!=errorlogobject.offset)
165 {
166 ErrorLogX errorlogx;
167 latlong_t errorlat=NO_LATLONG,errorlon=NO_LATLONG;
168
169 /* Calculate suitable coordinates */
170
171 if(nerrorlogobjects==1)
172 {
173 if(errorlogobjects[0].type=='N')
174 {
175 node_t node=(node_t)errorlogobjects[0].id;
176
177 lookup_lat_long_node(nodesx,node,&errorlat,&errorlon);
178 }
179 else if(errorlogobjects[0].type=='W')
180 {
181 way_t way=(way_t)errorlogobjects[0].id;
182
183 lookup_lat_long_way(waysx,nodesx,way,&errorlat,&errorlon,errorlogsx->number);
184 }
185 else if(errorlogobjects[0].type=='R')
186 {
187 relation_t relation=(relation_t)errorlogobjects[0].type;
188
189 lookup_lat_long_relation(relationsx,waysx,nodesx,relation,&errorlat,&errorlon,errorlogsx->number);
190 }
191 }
192 else
193 {
194 latlong_t latitude[8],longitude[8];
195 int i;
196 int ncoords=0,nnodes=0,nways=0,nrelations=0;
197
198 for(i=0;i<nerrorlogobjects;i++)
199 {
200 if(errorlogobjects[i].type=='N')
201 {
202 node_t node=(node_t)errorlogobjects[i].id;
203
204 if(lookup_lat_long_node(nodesx,node,&latitude[ncoords],&longitude[ncoords]))
205 ncoords++;
206
207 nnodes++;
208 }
209 else if(errorlogobjects[i].type=='W')
210 nways++;
211 else if(errorlogobjects[i].type=='R')
212 nrelations++;
213 }
214
215 if(nways==0 && nrelations==0) /* only nodes */
216 ;
217 else if(ncoords) /* some good nodes, possibly ways and/or relations */
218 ;
219 else if(nways) /* no good nodes, possibly some good ways */
220 {
221 for(i=0;i<nerrorlogobjects;i++)
222 if(errorlogobjects[i].type=='W')
223 {
224 way_t way=(way_t)errorlogobjects[i].id;
225
226 if(lookup_lat_long_way(waysx,nodesx,way,&latitude[ncoords],&longitude[ncoords],errorlogsx->number))
227 ncoords++;
228 }
229 }
230
231 if(nrelations==0) /* only nodes and/or ways */
232 ;
233 else if(ncoords) /* some good nodes and/or ways, possibly relations */
234 ;
235 else /* if(nrelations) */
236 {
237 for(i=0;i<nerrorlogobjects;i++)
238 if(errorlogobjects[i].type=='R')
239 {
240 relation_t relation=(relation_t)errorlogobjects[i].id;
241
242 if(lookup_lat_long_relation(relationsx,waysx,nodesx,relation,&latitude[ncoords],&longitude[ncoords],errorlogsx->number))
243 ncoords++;
244 }
245 }
246
247 if(ncoords)
248 {
249 errorlat=0;
250 errorlon=0;
251
252 for(i=0;i<ncoords;i++)
253 {
254 errorlat+=latitude[i];
255 errorlon+=longitude[i];
256 }
257
258 errorlat/=ncoords;
259 errorlon/=ncoords;
260 }
261 else
262 {
263 errorlat=NO_LATLONG;
264 errorlon=NO_LATLONG;
265 }
266 }
267
268 /* Write to file */
269
270 errorlogx.offset=offset;
271 errorlogx.length=errorlogobject.offset-offset;
272
273 errorlogx.latitude =errorlat;
274 errorlogx.longitude=errorlon;
275
276 WriteFileBuffered(newfd,&errorlogx,sizeof(ErrorLogX));
277
278 errorlogsx->number++;
279
280 offset=errorlogobject.offset;
281 nerrorlogobjects=0;
282
283 if(!(errorlogsx->number%10000))
284 printf_middle("Calculating Coordinates: Errors=%"Pindex_t,errorlogsx->number);
285 }
286
287 /* Store for later */
288
289 logassert(nerrorlogobjects<8,"Too many error log objects for one error message."); /* Only a limited amount of information stored. */
290
291 errorlogobjects[nerrorlogobjects]=errorlogobject;
292
293 nerrorlogobjects++;
294 }
295 while(!finished);
296
297 /* Unmap from memory / close the files */
298
299 #if !SLIM
300 nodesx->data=UnmapFile(nodesx->data);
301 #else
302 nodesx->fd=SlimUnmapFile(nodesx->fd);
303 #endif
304
305 waysx->fd=CloseFileBuffered(waysx->fd);
306 relationsx->rrfd=CloseFileBuffered(relationsx->rrfd);
307 relationsx->trfd=CloseFileBuffered(relationsx->trfd);
308
309 CloseFileBuffered(oldfd);
310 CloseFileBuffered(newfd);
311
312 /* Print the final message */
313
314 printf_last("Calculated Coordinates: Errors=%"Pindex_t,errorlogsx->number);
315 }
316
317
318 /*++++++++++++++++++++++++++++++++++++++
319 Re-index the nodes that were kept.
320
321 NodesX *nodesx The set of nodes to process (contains the filename and number of nodes).
322 ++++++++++++++++++++++++++++++++++++++*/
323
324 static void reindex_nodes(NodesX *nodesx)
325 {
326 int fd;
327 index_t index=0;
328 NodeX nodex;
329
330 nodesx->number=nodesx->knumber;
331
332 nodesx->idata=(node_t*)malloc(nodesx->number*sizeof(node_t));
333 log_malloc(nodesx->idata,nodesx->number*sizeof(node_t));
334
335 /* Get the node id for each node in the file. */
336
337 fd=ReOpenFileBuffered(nodesx->filename);
338
339 while(!ReadFileBuffered(fd,&nodex,sizeof(NodeX)))
340 {
341 nodesx->idata[index]=nodex.id;
342
343 index++;
344 }
345
346 CloseFileBuffered(fd);
347 }
348
349
350 /*++++++++++++++++++++++++++++++++++++++
351 Re-index the ways that were kept.
352
353 WaysX *waysx The set of ways to process (contains the filename and number of ways).
354 ++++++++++++++++++++++++++++++++++++++*/
355
356 static void reindex_ways(WaysX *waysx)
357 {
358 FILESORT_VARINT waysize;
359 int fd;
360 offset_t position=0;
361 index_t index=0;
362
363 waysx->number=waysx->knumber;
364
365 waysx->idata=(way_t*)malloc(waysx->number*sizeof(way_t));
366 waysx->odata=(offset_t*)malloc(waysx->number*sizeof(offset_t));
367
368 log_malloc(waysx->idata,waysx->number*sizeof(way_t));
369 log_malloc(waysx->odata,waysx->number*sizeof(offset_t));
370
371 /* Get the way id and the offset for each way in the file */
372
373 fd=ReOpenFileBuffered(waysx->filename);
374
375 while(!ReadFileBuffered(fd,&waysize,FILESORT_VARSIZE))
376 {
377 WayX wayx;
378
379 ReadFileBuffered(fd,&wayx,sizeof(WayX));
380
381 waysx->idata[index]=wayx.id;
382 waysx->odata[index]=position+FILESORT_VARSIZE+sizeof(WayX);
383
384 index++;
385
386 SkipFileBuffered(fd,waysize-sizeof(WayX));
387
388 position+=waysize+FILESORT_VARSIZE;
389 }
390
391 CloseFileBuffered(fd);
392 }
393
394
395 /*++++++++++++++++++++++++++++++++++++++
396 Re-index the relations that were kept.
397
398 RelationsX *relationsx The set of relations to process (contains the filenames and numbers of relations).
399 ++++++++++++++++++++++++++++++++++++++*/
400
401 static void reindex_relations(RelationsX *relationsx)
402 {
403 FILESORT_VARINT relationsize;
404 int fd;
405 offset_t position=0;
406 index_t index;
407 TurnRelX turnrelx;
408
409 /* Route relations */
410
411 relationsx->rrnumber=relationsx->rrknumber;
412
413 relationsx->rridata=(relation_t*)malloc(relationsx->rrnumber*sizeof(relation_t));
414 relationsx->rrodata=(offset_t*)malloc(relationsx->rrnumber*sizeof(offset_t));
415
416 log_malloc(relationsx->rridata,relationsx->rrnumber*sizeof(relation_t));
417 log_malloc(relationsx->rrodata,relationsx->rrnumber*sizeof(offset_t));
418
419 /* Get the relation id and the offset for each relation in the file */
420
421 fd=ReOpenFileBuffered(relationsx->rrfilename);
422
423 index=0;
424
425 while(!ReadFileBuffered(fd,&relationsize,FILESORT_VARSIZE))
426 {
427 RouteRelX routerelx;
428
429 ReadFileBuffered(fd,&routerelx,sizeof(RouteRelX));
430
431 relationsx->rridata[index]=routerelx.id;
432 relationsx->rrodata[index]=position+FILESORT_VARSIZE+sizeof(RouteRelX);
433
434 index++;
435
436 SkipFileBuffered(fd,relationsize-sizeof(RouteRelX));
437
438 position+=relationsize+FILESORT_VARSIZE;
439 }
440
441 CloseFileBuffered(fd);
442
443
444 /* Turn relations */
445
446 relationsx->trnumber=relationsx->trknumber;
447
448 relationsx->tridata=(relation_t*)malloc(relationsx->trnumber*sizeof(relation_t));
449
450 log_malloc(relationsx->tridata,relationsx->trnumber*sizeof(relation_t));
451
452 /* Get the relation id and the offset for each relation in the file */
453
454 fd=ReOpenFileBuffered(relationsx->trfilename);
455
456 index=0;
457
458 while(!ReadFileBuffered(fd,&turnrelx,sizeof(TurnRelX)))
459 {
460 relationsx->tridata[index]=turnrelx.id;
461
462 index++;
463 }
464
465 CloseFileBuffered(fd);
466 }
467
468
469 /*++++++++++++++++++++++++++++++++++++++
470 Lookup a node's latitude and longitude.
471
472 int lookup_lat_long_node Returns 1 if a node was found.
473
474 NodesX *nodesx The set of nodes to use.
475
476 node_t node The node number.
477
478 latlong_t *latitude Returns the latitude.
479
480 latlong_t *longitude Returns the longitude.
481 ++++++++++++++++++++++++++++++++++++++*/
482
483 static int lookup_lat_long_node(NodesX *nodesx,node_t node,latlong_t *latitude,latlong_t *longitude)
484 {
485 index_t index=IndexNodeX(nodesx,node);
486
487 if(index==NO_NODE)
488 return 0;
489 else
490 {
491 NodeX *nodex=LookupNodeX(nodesx,index,1);
492
493 *latitude =nodex->latitude;
494 *longitude=nodex->longitude;
495
496 return 1;
497 }
498 }
499
500
501 /*++++++++++++++++++++++++++++++++++++++
502 Lookup a way's latitude and longitude.
503
504 int lookup_lat_long_way Returns 1 if a way was found.
505
506 WaysX *waysx The set of ways to use.
507
508 NodesX *nodesx The set of nodes to use.
509
510 way_t way The way number.
511
512 latlong_t *latitude Returns the latitude.
513
514 latlong_t *longitude Returns the longitude.
515
516 index_t error The index of the error in the complete set of errors.
517 ++++++++++++++++++++++++++++++++++++++*/
518
519 static int lookup_lat_long_way(WaysX *waysx,NodesX *nodesx,way_t way,latlong_t *latitude,latlong_t *longitude,index_t error)
520 {
521 index_t index=IndexWayX(waysx,way);
522
523 if(index==NO_WAY)
524 return 0;
525 else
526 {
527 int count=1;
528 offset_t offset=waysx->odata[index];
529 node_t node1,node2,prevnode,node;
530 latlong_t latitude1,longitude1,latitude2,longitude2;
531
532 SeekFileBuffered(waysx->fd,offset);
533
534 /* Choose a random pair of adjacent nodes */
535
536 if(ReadFileBuffered(waysx->fd,&node1,sizeof(node_t)) || node1==NO_NODE_ID)
537 return 0;
538
539 if(ReadFileBuffered(waysx->fd,&node2,sizeof(node_t)) || node2==NO_NODE_ID)
540 return lookup_lat_long_node(nodesx,node1,latitude,longitude);
541
542 prevnode=node2;
543
544 while(!ReadFileBuffered(waysx->fd,&node,sizeof(node_t)) && node!=NO_NODE_ID)
545 {
546 count++;
547
548 if((error%count)==0) /* A 1/count chance */
549 {
550 node1=prevnode;
551 node2=node;
552 }
553
554 prevnode=node;
555 }
556
557 if(!lookup_lat_long_node(nodesx,node1,&latitude1,&longitude1))
558 return lookup_lat_long_node(nodesx,node2,latitude,longitude);
559
560 if(!lookup_lat_long_node(nodesx,node2,&latitude2,&longitude2))
561 return lookup_lat_long_node(nodesx,node1,latitude,longitude);
562
563 *latitude =(latitude1 +latitude2 )/2;
564 *longitude=(longitude1+longitude2)/2;
565
566 return 1;
567 }
568 }
569
570
571 /*++++++++++++++++++++++++++++++++++++++
572 Lookup a relation's latitude and longitude.
573
574 int lookup_lat_long_relation Returns 1 if a relation was found.
575
576 RelationsX *relationsx The set of relations to use.
577
578 WaysX *waysx The set of ways to use.
579
580 NodesX *nodesx The set of nodes to use.
581
582 relation_t relation The relation number.
583
584 latlong_t *latitude Returns the latitude.
585
586 latlong_t *longitude Returns the longitude.
587
588 index_t error The index of the error in the complete set of errors.
589 ++++++++++++++++++++++++++++++++++++++*/
590
591 static int lookup_lat_long_relation(RelationsX *relationsx,WaysX *waysx,NodesX *nodesx,relation_t relation,latlong_t *latitude,latlong_t *longitude,index_t error)
592 {
593 index_t index=IndexRouteRelX(relationsx,relation);
594
595 if(index==NO_RELATION)
596 {
597 index=IndexTurnRelX(relationsx,relation);
598
599 if(index==NO_RELATION)
600 return 0;
601 else
602 {
603 TurnRelX turnrelx;
604
605 SeekFileBuffered(relationsx->trfd,index*sizeof(TurnRelX));
606 ReadFileBuffered(relationsx->trfd,&turnrelx,sizeof(TurnRelX));
607
608 if(lookup_lat_long_node(nodesx,turnrelx.via,latitude,longitude))
609 return 1;
610
611 if(lookup_lat_long_way(waysx,nodesx,turnrelx.from,latitude,longitude,error))
612 return 1;
613
614 if(lookup_lat_long_way(waysx,nodesx,turnrelx.to,latitude,longitude,error))
615 return 1;
616
617 return 0;
618 }
619 }
620 else
621 {
622 int count;
623 offset_t offset=relationsx->rrodata[index];
624 node_t node=NO_NODE_ID,tempnode;
625 way_t way=NO_WAY_ID,tempway;
626 relation_t relation=NO_RELATION_ID,temprelation;
627
628 SeekFileBuffered(relationsx->rrfd,offset);
629
630 /* Choose a random node */
631
632 count=0;
633
634 while(!ReadFileBuffered(relationsx->rrfd,&tempnode,sizeof(node_t)) && tempnode!=NO_NODE_ID)
635 {
636 count++;
637
638 if((error%count)==0) /* A 1/count chance */
639 node=tempnode;
640 }
641
642 if(lookup_lat_long_node(nodesx,node,latitude,longitude))
643 return 1;
644
645 /* Choose a random way */
646
647 count=0;
648
649 while(!ReadFileBuffered(relationsx->rrfd,&tempway,sizeof(way_t)) && tempway!=NO_WAY_ID)
650 {
651 count++;
652
653 if((error%count)==0) /* A 1/count chance */
654 way=tempway;
655 }
656
657 if(lookup_lat_long_way(waysx,nodesx,way,latitude,longitude,error))
658 return 1;
659
660 /* Choose a random relation */
661
662 count=0;
663
664 while(!ReadFileBuffered(relationsx->rrfd,&temprelation,sizeof(relation_t)) && temprelation!=NO_RELATION_ID)
665 {
666 count++;
667
668 if((error%count)==0) /* A 1/count chance */
669 relation=temprelation;
670 }
671
672 return lookup_lat_long_relation(relationsx,waysx,nodesx,relation,latitude,longitude,error);
673 }
674 }
675
676
677 /*++++++++++++++++++++++++++++++++++++++
678 Sort the error logs geographically.
679
680 ErrorLogsX *errorlogsx The set of error logs to sort.
681 ++++++++++++++++++++++++++++++++++++++*/
682
683 void SortErrorLogsGeographically(ErrorLogsX *errorlogsx)
684 {
685 int oldfd,newfd;
686 ll_bin_t lat_min_bin,lat_max_bin,lon_min_bin,lon_max_bin;
687
688 /* Print the start message */
689
690 printf_first("Sorting Errors Geographically");
691
692 /* Work out the range of data */
693
694 lat_min=radians_to_latlong( 2);
695 lat_max=radians_to_latlong(-2);
696 lon_min=radians_to_latlong( 4);
697 lon_max=radians_to_latlong(-4);
698
699 /* Re-open the file read-only and a new file writeable */
700
701 newfd=ReplaceFileBuffered(errorbinfilename,&oldfd);
702
703 /* Sort errors geographically */
704
705 filesort_fixed(oldfd,newfd,sizeof(ErrorLogX),NULL,
706 (int (*)(const void*,const void*))sort_by_lat_long,
707 (int (*)(void*,index_t))measure_lat_long);
708
709 /* Close the files */
710
711 CloseFileBuffered(oldfd);
712 CloseFileBuffered(newfd);
713
714 /* Work out the number of bins */
715
716 lat_min_bin=latlong_to_bin(lat_min);
717 lon_min_bin=latlong_to_bin(lon_min);
718 lat_max_bin=latlong_to_bin(lat_max);
719 lon_max_bin=latlong_to_bin(lon_max);
720
721 errorlogsx->latzero=lat_min_bin;
722 errorlogsx->lonzero=lon_min_bin;
723
724 errorlogsx->latbins=(lat_max_bin-lat_min_bin)+1;
725 errorlogsx->lonbins=(lon_max_bin-lon_min_bin)+1;
726
727 /* Print the final message */
728
729 printf_last("Sorted Errors Geographically: Errors=%"Pindex_t,errorlogsx->number);
730 }
731
732
733 /*++++++++++++++++++++++++++++++++++++++
734 Sort the errors into latitude and longitude order (first by longitude bin
735 number, then by latitude bin number and then by exact longitude and then by
736 exact latitude).
737
738 int sort_by_lat_long Returns the comparison of the latitude and longitude fields.
739
740 ErrorLogX *a The first error location.
741
742 ErrorLogX *b The second error location.
743 ++++++++++++++++++++++++++++++++++++++*/
744
745 static int sort_by_lat_long(ErrorLogX *a,ErrorLogX *b)
746 {
747 ll_bin_t a_lon=latlong_to_bin(a->longitude);
748 ll_bin_t b_lon=latlong_to_bin(b->longitude);
749
750 if(a_lon<b_lon)
751 return(-1);
752 else if(a_lon>b_lon)
753 return(1);
754 else
755 {
756 ll_bin_t a_lat=latlong_to_bin(a->latitude);
757 ll_bin_t b_lat=latlong_to_bin(b->latitude);
758
759 if(a_lat<b_lat)
760 return(-1);
761 else if(a_lat>b_lat)
762 return(1);
763 else
764 {
765 if(a->longitude<b->longitude)
766 return(-1);
767 else if(a->longitude>b->longitude)
768 return(1);
769 else
770 {
771 if(a->latitude<b->latitude)
772 return(-1);
773 else if(a->latitude>b->latitude)
774 return(1);
775 }
776
777 return(FILESORT_PRESERVE_ORDER(a,b));
778 }
779 }
780 }
781
782
783 /*++++++++++++++++++++++++++++++++++++++
784 Measure the extent of the data.
785
786 int measure_lat_long Return 1 if the value is to be kept, otherwise 0.
787
788 ErrorLogX *errorlogx The error location.
789
790 index_t index The number of sorted error locations that have already been written to the output file.
791 ++++++++++++++++++++++++++++++++++++++*/
792
793 static int measure_lat_long(ErrorLogX *errorlogx,index_t index)
794 {
795 if(errorlogx->latitude!=NO_LATLONG)
796 {
797 if(errorlogx->latitude<lat_min)
798 lat_min=errorlogx->latitude;
799 if(errorlogx->latitude>lat_max)
800 lat_max=errorlogx->latitude;
801 if(errorlogx->longitude<lon_min)
802 lon_min=errorlogx->longitude;
803 if(errorlogx->longitude>lon_max)
804 lon_max=errorlogx->longitude;
805 }
806
807 return(1);
808 }
809
810
811 /*++++++++++++++++++++++++++++++++++++++
812 Save the binary error log.
813
814 ErrorLogsX *errorlogsx The set of error logs to write.
815
816 char *filename The name of the final file to write.
817 ++++++++++++++++++++++++++++++++++++++*/
818
819 void SaveErrorLogs(ErrorLogsX *errorlogsx,char *filename)
820 {
821 ErrorLogsFile errorlogsfile;
822 ErrorLogX errorlogx;
823 int oldfd,newfd;
824 ll_bin2_t latlonbin=0,maxlatlonbins;
825 index_t *offsets;
826 index_t number=0,number_geo=0,number_nongeo=0;
827 offset_t size;
828
829 /* Print the start message */
830
831 printf_first("Writing Errors: Geographical=0 Non-geographical=0");
832
833 /* Allocate the memory for the geographical offsets array */
834
835 offsets=(index_t*)malloc((errorlogsx->latbins*errorlogsx->lonbins+1)*sizeof(index_t));
836
837 logassert(offsets,"Failed to allocate memory (try using slim mode?)"); /* Check malloc() worked */
838
839 latlonbin=0;
840
841 /* Re-open the file */
842
843 oldfd=ReOpenFileBuffered(errorbinfilename);
844
845 newfd=OpenFileBufferedNew(filename);
846
847 /* Write out the geographical errors */
848
849 SeekFileBuffered(newfd,sizeof(ErrorLogsFile)+(errorlogsx->latbins*errorlogsx->lonbins+1)*sizeof(index_t));
850
851 while(!ReadFileBuffered(oldfd,&errorlogx,sizeof(ErrorLogX)))
852 {
853 ErrorLog errorlog={0};
854 ll_bin_t latbin,lonbin;
855 ll_bin2_t llbin;
856
857 if(errorlogx.latitude==NO_LATLONG)
858 continue;
859
860 /* Create the ErrorLog */
861
862 errorlog.latoffset=latlong_to_off(errorlogx.latitude);
863 errorlog.lonoffset=latlong_to_off(errorlogx.longitude);
864
865 errorlog.offset=errorlogx.offset;
866 errorlog.length=errorlogx.length;
867
868 /* Work out the offsets */
869
870 latbin=latlong_to_bin(errorlogx.latitude )-errorlogsx->latzero;
871 lonbin=latlong_to_bin(errorlogx.longitude)-errorlogsx->lonzero;
872 llbin=lonbin*errorlogsx->latbins+latbin;
873
874 for(;latlonbin<=llbin;latlonbin++)
875 offsets[latlonbin]=number_geo;
876
877 /* Write the data */
878
879 WriteFileBuffered(newfd,&errorlog,sizeof(ErrorLog));
880
881 number_geo++;
882 number++;
883
884 if(!(number%10000))
885 printf_middle("Writing Errors: Geographical=%"Pindex_t" Non-geographical=%"Pindex_t,number_geo,number_nongeo);
886 }
887
888 /* Write out the non-geographical errors */
889
890 SeekFileBuffered(oldfd,0);
891
892 while(!ReadFileBuffered(oldfd,&errorlogx,sizeof(ErrorLogX)))
893 {
894 ErrorLog errorlog={0};
895
896 if(errorlogx.latitude!=NO_LATLONG)
897 continue;
898
899 /* Create the ErrorLog */
900
901 errorlog.latoffset=0;
902 errorlog.lonoffset=0;
903
904 errorlog.offset=errorlogx.offset;
905 errorlog.length=errorlogx.length;
906
907 /* Write the data */
908
909 WriteFileBuffered(newfd,&errorlog,sizeof(ErrorLog));
910
911 number_nongeo++;
912 number++;
913
914 if(!(number%10000))
915 printf_middle("Writing Errors: Geographical=%"Pindex_t" Non-geographical=%"Pindex_t,number_geo,number_nongeo);
916 }
917
918 /* Close the input file */
919
920 CloseFileBuffered(oldfd);
921
922 DeleteFile(errorbinfilename);
923
924 /* Append the text from the log file */
925
926 size=SizeFile(errorlogfilename);
927
928 oldfd=ReOpenFileBuffered(errorlogfilename);
929
930 while(size)
931 {
932 int i;
933 char buffer[4096];
934 offset_t chunksize=(size>(offset_t)sizeof(buffer)?(offset_t)sizeof(buffer):size);
935
936 ReadFileBuffered(oldfd,buffer,chunksize);
937
938 for(i=0;i<chunksize;i++)
939 if(buffer[i]=='\n')
940 buffer[i]=0;
941
942 WriteFileBuffered(newfd,buffer,chunksize);
943
944 size-=chunksize;
945 }
946
947 CloseFileBuffered(oldfd);
948
949 /* Finish off the offset indexing and write them out */
950
951 maxlatlonbins=errorlogsx->latbins*errorlogsx->lonbins;
952
953 for(;latlonbin<=maxlatlonbins;latlonbin++)
954 offsets[latlonbin]=number_geo;
955
956 SeekFileBuffered(newfd,sizeof(ErrorLogsFile));
957 WriteFileBuffered(newfd,offsets,(errorlogsx->latbins*errorlogsx->lonbins+1)*sizeof(index_t));
958
959 free(offsets);
960
961 /* Write out the header structure */
962
963 errorlogsfile.number =number;
964 errorlogsfile.number_geo =number_geo;
965 errorlogsfile.number_nongeo=number_nongeo;
966
967 errorlogsfile.latbins=errorlogsx->latbins;
968 errorlogsfile.lonbins=errorlogsx->lonbins;
969
970 errorlogsfile.latzero=errorlogsx->latzero;
971 errorlogsfile.lonzero=errorlogsx->lonzero;
972
973 SeekFileBuffered(newfd,0);
974 WriteFileBuffered(newfd,&errorlogsfile,sizeof(ErrorLogsFile));
975
976 CloseFileBuffered(newfd);
977
978 /* Print the final message */
979
980 printf_last("Wrote Errors: Geographical=%"Pindex_t" Non-geographical=%"Pindex_t,number_geo,number_nongeo);
981 }