trunk/src/errorlogx.c

/***************************************
 Error log processing functions.

 Part of the Routino routing software.
 ******************/ /******************
 This file Copyright 2013 Andrew M. Bishop

 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as published by
 the Free Software Foundation, either version 3 of the License, or
 (at your option) any later version.

 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU Affero General Public License for more details.

 You should have received a copy of the GNU Affero General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 ***************************************/


#include "typesx.h"
#include "nodesx.h"
#include "waysx.h"
#include "relationsx.h"

#include "files.h"
#include "errorlogx.h"
#include "sorting.h"


/* Local variables */

/*+ The name of the error log file. +*/
extern char *errorlogfilename;

/*+ The name of the binary error log file. +*/
extern char *errorbinfilename;


/* Local functions */

static void reindex_nodes(NodesX *nodesx);
static void reindex_ways(WaysX *waysx);
static void reindex_relations(RelationsX *relationsx);
static int sort_by_lat_long(ErrorLogX *a,ErrorLogX *b);


/*++++++++++++++++++++++++++++++++++++++
  Process the binary error log.

  NodesX *nodesx The set of nodes.

  WaysX *waysx The set of ways.

  RelationsX *relationsx The set of relations.
  ++++++++++++++++++++++++++++++++++++++*/

void ProcessErrorLogs(NodesX *nodesx,WaysX *waysx,RelationsX *relationsx)
{
 int oldfd,newfd;
 uint32_t offset=0;
 int nerrorlogobjects=0;
 int finished;
 ErrorLogObject errorlogobjects[8];
 index_t number;

 /* Re-index the nodes, ways and relations */

 printf_first("Re-indexing the Nodes, Ways and Relations: Nodes=0 Ways=0 Relations=0");

 reindex_nodes(nodesx);

 printf_middle("Re-indexing the Nodes, Ways and Relations: Nodes=%"Pindex_t" Ways=0 Relations=0",nodesx->number);

 reindex_ways(waysx);

 printf_middle("Re-indexing the Nodes, Ways and Relations: Nodes=%"Pindex_t" Ways=%"Pindex_t" Relations=0",nodesx->number,waysx->number);

 reindex_relations(relationsx);

 printf_last("Re-indexed the Nodes, Ways and Relations: Nodes=%"Pindex_t" Ways=%"Pindex_t" Relations=%"Pindex_t,nodesx->number,waysx->number,relationsx->trnumber);


 /* Print the start message */

 printf_first("Calculating Coordinates: Errors=0");

 /* Map into memory / open the files */

#if !SLIM
 nodesx->data=MapFile(nodesx->filename);
 waysx->data=MapFile(waysx->filename);
#else
 nodesx->fd=ReOpenFile(nodesx->filename);
 waysx->fd=ReOpenFile(waysx->filename);

 InvalidateNodeXCache(nodesx->cache);
 InvalidateWayXCache(waysx->cache);
#endif

 /* Open the binary log file read-only and a new file writeable */

 oldfd=ReOpenFile(errorbinfilename);

 DeleteFile(errorbinfilename);

 newfd=OpenFileNew(errorbinfilename);

 /* Loop through the file and merge the raw data into coordinates */

 number=0;

 do
   {
    ErrorLogObject errorlogobject;

    finished=ReadFile(oldfd,&errorlogobject,sizeof(ErrorLogObject));

    if(finished)
       errorlogobject.offset=SizeFile(errorlogfilename);

    if(offset!=errorlogobject.offset)
      {
       ErrorLogX errorlogx;
       int i;
       latlong_t latitude=NO_LATLONG,longitude=NO_LATLONG;

       /* Calculate suitable coordinates */

       for(i=0;i<nerrorlogobjects;i++)
         {
          if(((errorlogobjects[i].type_id>>56)&0xff)=='N')
            {
             index_t node=IndexNodeX(nodesx,errorlogobjects[i].type_id&(uint64_t)0x00ffffffffffffff);

             if(node!=NO_NODE)
               {
                NodeX *nodex=LookupNodeX(nodesx,node,1);

                latitude =nodex->latitude;
                longitude=nodex->longitude;
               }
            }
         }

       /* Write to file */

       errorlogx.offset=offset;
       errorlogx.length=errorlogobject.offset-offset;

       errorlogx.latitude =latitude;
       errorlogx.longitude=longitude;

       //       printf("errorlogx.lat=%ld (%f) .lon=%ld (%f)\n",errorlogx.latitude,radians_to_degrees(latlong_to_radians(errorlogx.latitude)),errorlogx.longitude,radians_to_degrees(latlong_to_radians(errorlogx.longitude)));

       WriteFile(newfd,&errorlogx,sizeof(ErrorLogX));

       number++;

       offset=errorlogobject.offset;
       nerrorlogobjects=0;

       if(!(number%10000))
          printf_middle("Calculating Coordinates: Errors=%"Pindex_t,number);
      }

    /* Store for later */

    logassert(nerrorlogobjects<8,"Too many error log objects for one error message."); /* Only a limited amount of information stored. */

    errorlogobjects[nerrorlogobjects]=errorlogobject;

    nerrorlogobjects++;
   }
 while(!finished);

 /* Unmap from memory / close the files */

#if !SLIM
 nodesx->data=UnmapFile(nodesx->data);
 waysx->data=UnmapFile(waysx->data);
#else
 nodesx->fd=CloseFile(nodesx->fd);
 waysx->fd=CloseFile(waysx->fd);
#endif

 CloseFile(oldfd);
 CloseFile(newfd);

 /* Print the final message */

 printf_last("Calculated Coordinates: Errors=%"Pindex_t,number);
}


/*++++++++++++++++++++++++++++++++++++++
  Re-index the nodes that were kept.

  NodesX *nodesx The set of nodes to process (contains the filename and number of nodes).
  ++++++++++++++++++++++++++++++++++++++*/

static void reindex_nodes(NodesX *nodesx)
{
 int fd;
 index_t index=0;
 NodeX nodex;

 nodesx->number=nodesx->knumber;

 nodesx->idata=(node_t*)malloc(nodesx->number*sizeof(node_t));

 /* Get the node id for each node in the file. */

 fd=ReOpenFile(nodesx->filename);

 while(!ReadFile(fd,&nodex,sizeof(NodeX)))
   {
    nodesx->idata[index]=nodex.id;

    index++;
   }

 CloseFile(fd);
}


/*++++++++++++++++++++++++++++++++++++++
  Re-index the ways that were kept.

  WaysX *waysx The set of ways to process (contains the filename and number of ways).
  ++++++++++++++++++++++++++++++++++++++*/

static void reindex_ways(WaysX *waysx)
{
 int fd;
 off_t size,position=0;
 index_t index=0;
 WayX wayx;

 waysx->number=waysx->knumber;

 waysx->idata=(way_t*)malloc(waysx->number*sizeof(way_t));
 waysx->odata=(off_t*)malloc(waysx->number*sizeof(off_t));

 /* Get the way id and the offset for each way in the file */

 size=SizeFile(waysx->filename);

 fd=ReOpenFile(waysx->filename);

 while(position<size)
   {
    FILESORT_VARINT waysize;

    SeekReadFile(fd,&waysize,FILESORT_VARSIZE,position);
    ReadFile(fd,&wayx,sizeof(WayX));

    waysx->idata[index]=wayx.id;
    waysx->odata[index]=position;

    index++;

    position+=waysize+FILESORT_VARSIZE;
   }

 CloseFile(fd);
}


/*++++++++++++++++++++++++++++++++++++++
  Re-index the relations that were kept.

  RelationsX *relationsx The set of relations to process (contains the filename and number of relations).
  ++++++++++++++++++++++++++++++++++++++*/

static void reindex_relations(RelationsX *relationsx)
{
}


/*++++++++++++++++++++++++++++++++++++++
  Sort the error logs geographically.
  ++++++++++++++++++++++++++++++++++++++*/

void SortErrorLogsGeographically(void)
{
 int oldfd,newfd;

 /* Print the start message */

 printf_first("Sorting Errors Geographically");

 /* Re-open the file read-only and a new file writeable */

 oldfd=ReOpenFile(errorbinfilename);

 DeleteFile(errorbinfilename);

 newfd=OpenFileNew(errorbinfilename);

 /* Sort errors geographically */

 filesort_fixed(oldfd,newfd,sizeof(ErrorLogX),NULL,
                                              (int (*)(const void*,const void*))sort_by_lat_long,
                                              NULL);

 /* Close the files */

 CloseFile(oldfd);
 CloseFile(newfd);

 /* Print the final message */

 printf_last("Sorted Errors Geographically");
}


/*++++++++++++++++++++++++++++++++++++++
  Sort the errors into latitude and longitude order (first by longitude bin
  number, then by latitude bin number and then by exact longitude and then by
  exact latitude).

  int sort_by_lat_long Returns the comparison of the latitude and longitude fields.

  ErrorLogX *a The first error location.

  ErrorLogX *b The second error location.
  ++++++++++++++++++++++++++++++++++++++*/

static int sort_by_lat_long(ErrorLogX *a,ErrorLogX *b)
{
 ll_bin_t a_lon=latlong_to_bin(a->longitude);
 ll_bin_t b_lon=latlong_to_bin(b->longitude);

 if(a_lon<b_lon)
    return(-1);
 else if(a_lon>b_lon)
    return(1);
 else
   {
    ll_bin_t a_lat=latlong_to_bin(a->latitude);
    ll_bin_t b_lat=latlong_to_bin(b->latitude);

    if(a_lat<b_lat)
       return(-1);
    else if(a_lat>b_lat)
       return(1);
    else
      {
       if(a->longitude<b->longitude)
          return(-1);
       else if(a->longitude>b->longitude)
          return(1);
       else
         {
          if(a->latitude<b->latitude)
             return(-1);
          else if(a->latitude>b->latitude)
             return(1);
         }

       return(FILESORT_PRESERVE_ORDER(a,b));
      }
   }
}


/*++++++++++++++++++++++++++++++++++++++
  Save the binary error log.

  NodesX *nodesx The set of nodes to copy some data from.

  char *filename The name of the final file to write.
  ++++++++++++++++++++++++++++++++++++++*/

void SaveErrorLogs(NodesX *nodesx,char *filename)
{
 ErrorLogsFile errorlogsfile;
 ErrorLogX errorlogx;
 int oldfd,newfd;
 ll_bin2_t latlonbin=0,maxlatlonbins;
 index_t *offsets;
 index_t number=0,number_geo=0,number_nongeo=0;
 off_t size;

 /* Print the start message */

 printf_first("Writing Errors: Geographical=0 Non-geographical=0");

 /* Allocate the memory for the geographical offsets array */

 offsets=(index_t*)malloc((nodesx->latbins*nodesx->lonbins+1)*sizeof(index_t));

 logassert(offsets,"Failed to allocate memory (try using slim mode?)"); /* Check malloc() worked */

 latlonbin=0;

 /* Re-open the file */

 oldfd=ReOpenFile(errorbinfilename);

 newfd=OpenFileNew(filename);

 /* Write out the geographical errors */

 SeekFile(newfd,sizeof(ErrorLogsFile)+(nodesx->latbins*nodesx->lonbins+1)*sizeof(index_t));

 while(!ReadFile(oldfd,&errorlogx,sizeof(ErrorLogX)))
   {
    ErrorLog errorlog={0};
    ll_bin_t latbin,lonbin;
    ll_bin2_t llbin;

    if(errorlogx.latitude==NO_LATLONG)
       continue;

    /* Create the ErrorLog */

    errorlog.latoffset=latlong_to_off(errorlogx.latitude);
    errorlog.lonoffset=latlong_to_off(errorlogx.longitude);

    errorlog.offset=errorlogx.offset;
    errorlog.length=errorlogx.length;

    /* Work out the offsets */

    latbin=latlong_to_bin(errorlogx.latitude )-nodesx->latzero;
    lonbin=latlong_to_bin(errorlogx.longitude)-nodesx->lonzero;
    llbin=lonbin*nodesx->latbins+latbin;

    for(;latlonbin<=llbin;latlonbin++)
       offsets[latlonbin]=number_geo;

    /* Write the data */

    WriteFile(newfd,&errorlog,sizeof(ErrorLog));

    number_geo++;
    number++;

    if(!(number%10000))
       printf_middle("Writing Errors: Geographical=%"Pindex_t" Non-geographical=%"Pindex_t,number_geo,number_nongeo);
   }

 /* Write out the non-geographical errors */

 SeekFile(oldfd,0);

 while(!ReadFile(oldfd,&errorlogx,sizeof(ErrorLogX)))
   {
    ErrorLog errorlog={0};

    if(errorlogx.latitude!=NO_LATLONG)
       continue;

    /* Create the ErrorLog */

    errorlog.latoffset=0;
    errorlog.lonoffset=0;

    errorlog.offset=errorlogx.offset;
    errorlog.length=errorlogx.length;

    /* Write the data */

    WriteFile(newfd,&errorlog,sizeof(ErrorLog));

    number_nongeo++;
    number++;

    if(!(number%10000))
       printf_middle("Writing Errors: Geographical=%"Pindex_t" Non-geographical=%"Pindex_t,number_geo,number_nongeo);
   }

 /* Close the input file */

 CloseFile(oldfd);

 DeleteFile(errorbinfilename);

 /* Append the text from the log file */

 size=SizeFile(errorlogfilename);

 oldfd=ReOpenFile(errorlogfilename);

 while(size)
   {
    char *buffer[4096];
    off_t chunksize=(size>sizeof(buffer)?sizeof(buffer):size);

    ReadFile(oldfd,buffer,chunksize);

    WriteFile(newfd,buffer,chunksize);

    size-=chunksize;
   }

 CloseFile(oldfd);

 /* Finish off the offset indexing and write them out */

 maxlatlonbins=nodesx->latbins*nodesx->lonbins;

 for(;latlonbin<=maxlatlonbins;latlonbin++)
    offsets[latlonbin]=number_geo;

 SeekFile(newfd,sizeof(ErrorLogsFile));
 WriteFile(newfd,offsets,(nodesx->latbins*nodesx->lonbins+1)*sizeof(index_t));

 free(offsets);

 /* Write out the header structure */

 errorlogsfile.number       =number;
 errorlogsfile.number_geo   =number_geo;
 errorlogsfile.number_nongeo=number_nongeo;

 errorlogsfile.latbins=nodesx->latbins;
 errorlogsfile.lonbins=nodesx->lonbins;

 errorlogsfile.latzero=nodesx->latzero;
 errorlogsfile.lonzero=nodesx->lonzero;

 SeekFile(newfd,0);
 WriteFile(newfd,&errorlogsfile,sizeof(ErrorLogsFile));

 CloseFile(newfd);

 /* Print the final message */

 printf_last("Wrote Errors: Geographical=%"Pindex_t" Non-geographical=%"Pindex_t,number_geo,number_nongeo);
}
1	amb	1317	/***************************************
2	amb	1319	Error log processing functions.
3	amb	1317
4			Part of the Routino routing software.
5			****************/ /****************
6			This file Copyright 2013 Andrew M. Bishop
7
8			This program is free software: you can redistribute it and/or modify
9			it under the terms of the GNU Affero General Public License as published by
10			the Free Software Foundation, either version 3 of the License, or
11			(at your option) any later version.
12
13			This program is distributed in the hope that it will be useful,
14			but WITHOUT ANY WARRANTY; without even the implied warranty of
15			MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16			GNU Affero General Public License for more details.
17
18			You should have received a copy of the GNU Affero General Public License
19			along with this program. If not, see <http://www.gnu.org/licenses/>.
20			***************************************/
21
22
23			#include "typesx.h"
24			#include "nodesx.h"
25			#include "waysx.h"
26			#include "relationsx.h"
27
28			#include "files.h"
29	amb	1319	#include "errorlogx.h"
30	amb	1317	#include "sorting.h"
31
32
33			/* Local variables */
34
35			/+ The name of the error log file. +/
36			extern char *errorlogfilename;
37
38			/+ The name of the binary error log file. +/
39			extern char *errorbinfilename;
40
41
42			/* Local functions */
43
44			static void reindex_nodes(NodesX *nodesx);
45			static void reindex_ways(WaysX *waysx);
46			static void reindex_relations(RelationsX *relationsx);
47			static int sort_by_lat_long(ErrorLogX a,ErrorLogX b);
48
49
50			/*++++++++++++++++++++++++++++++++++++++
51			Process the binary error log.
52
53			NodesX *nodesx The set of nodes.
54
55			WaysX *waysx The set of ways.
56
57			RelationsX *relationsx The set of relations.
58			++++++++++++++++++++++++++++++++++++++*/
59
60			void ProcessErrorLogs(NodesX nodesx,WaysX waysx,RelationsX *relationsx)
61			{
62			int oldfd,newfd;
63			uint32_t offset=0;
64			int nerrorlogobjects=0;
65			int finished;
66			ErrorLogObject errorlogobjects[8];
67			index_t number;
68
69			/* Re-index the nodes, ways and relations */
70
71			printf_first("Re-indexing the Nodes, Ways and Relations: Nodes=0 Ways=0 Relations=0");
72
73			reindex_nodes(nodesx);
74
75			printf_middle("Re-indexing the Nodes, Ways and Relations: Nodes=%"Pindex_t" Ways=0 Relations=0",nodesx->number);
76
77			reindex_ways(waysx);
78
79			printf_middle("Re-indexing the Nodes, Ways and Relations: Nodes=%"Pindex_t" Ways=%"Pindex_t" Relations=0",nodesx->number,waysx->number);
80
81			reindex_relations(relationsx);
82
83			printf_last("Re-indexed the Nodes, Ways and Relations: Nodes=%"Pindex_t" Ways=%"Pindex_t" Relations=%"Pindex_t,nodesx->number,waysx->number,relationsx->trnumber);
84
85
86			/* Print the start message */
87
88			printf_first("Calculating Coordinates: Errors=0");
89
90			/* Map into memory / open the files */
91
92			#if !SLIM
93			nodesx->data=MapFile(nodesx->filename);
94			waysx->data=MapFile(waysx->filename);
95			#else
96			nodesx->fd=ReOpenFile(nodesx->filename);
97			waysx->fd=ReOpenFile(waysx->filename);
98
99			InvalidateNodeXCache(nodesx->cache);
100			InvalidateWayXCache(waysx->cache);
101			#endif
102
103			/* Open the binary log file read-only and a new file writeable */
104
105			oldfd=ReOpenFile(errorbinfilename);
106
107			DeleteFile(errorbinfilename);
108
109			newfd=OpenFileNew(errorbinfilename);
110
111			/* Loop through the file and merge the raw data into coordinates */
112
113			number=0;
114
115			do
116			{
117			ErrorLogObject errorlogobject;
118
119			finished=ReadFile(oldfd,&errorlogobject,sizeof(ErrorLogObject));
120
121			if(finished)
122			errorlogobject.offset=SizeFile(errorlogfilename);
123
124			if(offset!=errorlogobject.offset)
125			{
126			ErrorLogX errorlogx;
127			int i;
128			latlong_t latitude=NO_LATLONG,longitude=NO_LATLONG;
129
130			/* Calculate suitable coordinates */
131
132			for(i=0;i<nerrorlogobjects;i++)
133			{
134			if(((errorlogobjects[i].type_id>>56)&0xff)=='N')
135			{
136			index_t node=IndexNodeX(nodesx,errorlogobjects[i].type_id&(uint64_t)0x00ffffffffffffff);
137
138			if(node!=NO_NODE)
139			{
140			NodeX *nodex=LookupNodeX(nodesx,node,1);
141
142			latitude =nodex->latitude;
143			longitude=nodex->longitude;
144			}
145			}
146			}
147
148			/* Write to file */
149
150			errorlogx.offset=offset;
151			errorlogx.length=errorlogobject.offset-offset;
152
153			errorlogx.latitude =latitude;
154			errorlogx.longitude=longitude;
155
156			// printf("errorlogx.lat=%ld (%f) .lon=%ld (%f)\n",errorlogx.latitude,radians_to_degrees(latlong_to_radians(errorlogx.latitude)),errorlogx.longitude,radians_to_degrees(latlong_to_radians(errorlogx.longitude)));
157
158			WriteFile(newfd,&errorlogx,sizeof(ErrorLogX));
159
160			number++;
161
162			offset=errorlogobject.offset;
163			nerrorlogobjects=0;
164
165			if(!(number%10000))
166			printf_middle("Calculating Coordinates: Errors=%"Pindex_t,number);
167			}
168
169			/* Store for later */
170
171			logassert(nerrorlogobjects<8,"Too many error log objects for one error message."); /* Only a limited amount of information stored. */
172
173			errorlogobjects[nerrorlogobjects]=errorlogobject;
174
175			nerrorlogobjects++;
176			}
177			while(!finished);
178
179			/* Unmap from memory / close the files */
180
181			#if !SLIM
182			nodesx->data=UnmapFile(nodesx->data);
183			waysx->data=UnmapFile(waysx->data);
184			#else
185			nodesx->fd=CloseFile(nodesx->fd);
186			waysx->fd=CloseFile(waysx->fd);
187			#endif
188
189			CloseFile(oldfd);
190			CloseFile(newfd);
191
192			/* Print the final message */
193
194			printf_last("Calculated Coordinates: Errors=%"Pindex_t,number);
195			}
196
197
198			/*++++++++++++++++++++++++++++++++++++++
199			Re-index the nodes that were kept.
200
201			NodesX *nodesx The set of nodes to process (contains the filename and number of nodes).
202			++++++++++++++++++++++++++++++++++++++*/
203
204			static void reindex_nodes(NodesX *nodesx)
205			{
206			int fd;
207			index_t index=0;
208			NodeX nodex;
209
210			nodesx->number=nodesx->knumber;
211
212			nodesx->idata=(node_t)malloc(nodesx->numbersizeof(node_t));
213
214			/* Get the node id for each node in the file. */
215
216			fd=ReOpenFile(nodesx->filename);
217
218			while(!ReadFile(fd,&nodex,sizeof(NodeX)))
219			{
220			nodesx->idata[index]=nodex.id;
221
222			index++;
223			}
224
225			CloseFile(fd);
226			}
227
228
229			/*++++++++++++++++++++++++++++++++++++++
230			Re-index the ways that were kept.
231
232			WaysX *waysx The set of ways to process (contains the filename and number of ways).
233			++++++++++++++++++++++++++++++++++++++*/
234
235			static void reindex_ways(WaysX *waysx)
236			{
237			int fd;
238			off_t size,position=0;
239			index_t index=0;
240			WayX wayx;
241
242			waysx->number=waysx->knumber;
243
244			waysx->idata=(way_t)malloc(waysx->numbersizeof(way_t));
245			waysx->odata=(off_t)malloc(waysx->numbersizeof(off_t));
246
247			/* Get the way id and the offset for each way in the file */
248
249			size=SizeFile(waysx->filename);
250
251			fd=ReOpenFile(waysx->filename);
252
253			while(position<size)
254			{
255			FILESORT_VARINT waysize;
256
257			SeekReadFile(fd,&waysize,FILESORT_VARSIZE,position);
258			ReadFile(fd,&wayx,sizeof(WayX));
259
260			waysx->idata[index]=wayx.id;
261			waysx->odata[index]=position;
262
263			index++;
264
265			position+=waysize+FILESORT_VARSIZE;
266			}
267
268			CloseFile(fd);
269			}
270
271
272			/*++++++++++++++++++++++++++++++++++++++
273			Re-index the relations that were kept.
274
275			RelationsX *relationsx The set of relations to process (contains the filename and number of relations).
276			++++++++++++++++++++++++++++++++++++++*/
277
278			static void reindex_relations(RelationsX *relationsx)
279			{
280			}
281
282
283			/*++++++++++++++++++++++++++++++++++++++
284			Sort the error logs geographically.
285			++++++++++++++++++++++++++++++++++++++*/
286
287			void SortErrorLogsGeographically(void)
288			{
289			int oldfd,newfd;
290
291			/* Print the start message */
292
293			printf_first("Sorting Errors Geographically");
294
295			/* Re-open the file read-only and a new file writeable */
296
297			oldfd=ReOpenFile(errorbinfilename);
298
299			DeleteFile(errorbinfilename);
300
301			newfd=OpenFileNew(errorbinfilename);
302
303			/* Sort errors geographically */
304
305			filesort_fixed(oldfd,newfd,sizeof(ErrorLogX),NULL,
306			(int ()(const void,const void*))sort_by_lat_long,
307			NULL);
308
309			/* Close the files */
310
311			CloseFile(oldfd);
312			CloseFile(newfd);
313
314			/* Print the final message */
315
316			printf_last("Sorted Errors Geographically");
317			}
318
319
320			/*++++++++++++++++++++++++++++++++++++++
321			Sort the errors into latitude and longitude order (first by longitude bin
322			number, then by latitude bin number and then by exact longitude and then by
323			exact latitude).
324
325			int sort_by_lat_long Returns the comparison of the latitude and longitude fields.
326
327			ErrorLogX *a The first error location.
328
329			ErrorLogX *b The second error location.
330			++++++++++++++++++++++++++++++++++++++*/
331
332			static int sort_by_lat_long(ErrorLogX a,ErrorLogX b)
333			{
334			ll_bin_t a_lon=latlong_to_bin(a->longitude);
335			ll_bin_t b_lon=latlong_to_bin(b->longitude);
336
337			if(a_lon<b_lon)
338			return(-1);
339			else if(a_lon>b_lon)
340			return(1);
341			else
342			{
343			ll_bin_t a_lat=latlong_to_bin(a->latitude);
344			ll_bin_t b_lat=latlong_to_bin(b->latitude);
345
346			if(a_lat<b_lat)
347			return(-1);
348			else if(a_lat>b_lat)
349			return(1);
350			else
351			{
352			if(a->longitude<b->longitude)
353			return(-1);
354			else if(a->longitude>b->longitude)
355			return(1);
356			else
357			{
358			if(a->latitude<b->latitude)
359			return(-1);
360			else if(a->latitude>b->latitude)
361			return(1);
362			}
363
364			return(FILESORT_PRESERVE_ORDER(a,b));
365			}
366			}
367			}
368
369
370			/*++++++++++++++++++++++++++++++++++++++
371			Save the binary error log.
372
373			NodesX *nodesx The set of nodes to copy some data from.
374
375			char *filename The name of the final file to write.
376			++++++++++++++++++++++++++++++++++++++*/
377
378			void SaveErrorLogs(NodesX nodesx,char filename)
379			{
380			ErrorLogsFile errorlogsfile;
381			ErrorLogX errorlogx;
382			int oldfd,newfd;
383			ll_bin2_t latlonbin=0,maxlatlonbins;
384			index_t *offsets;
385			index_t number=0,number_geo=0,number_nongeo=0;
386			off_t size;
387
388			/* Print the start message */
389
390			printf_first("Writing Errors: Geographical=0 Non-geographical=0");
391
392			/* Allocate the memory for the geographical offsets array */
393
394			offsets=(index_t)malloc((nodesx->latbinsnodesx->lonbins+1)*sizeof(index_t));
395
396			logassert(offsets,"Failed to allocate memory (try using slim mode?)"); /* Check malloc() worked */
397
398			latlonbin=0;
399
400			/* Re-open the file */
401
402			oldfd=ReOpenFile(errorbinfilename);
403
404			newfd=OpenFileNew(filename);
405
406			/* Write out the geographical errors */
407
408			SeekFile(newfd,sizeof(ErrorLogsFile)+(nodesx->latbinsnodesx->lonbins+1)sizeof(index_t));
409
410			while(!ReadFile(oldfd,&errorlogx,sizeof(ErrorLogX)))
411			{
412			ErrorLog errorlog={0};
413			ll_bin_t latbin,lonbin;
414			ll_bin2_t llbin;
415
416			if(errorlogx.latitude==NO_LATLONG)
417			continue;
418
419			/* Create the ErrorLog */
420
421			errorlog.latoffset=latlong_to_off(errorlogx.latitude);
422			errorlog.lonoffset=latlong_to_off(errorlogx.longitude);
423
424			errorlog.offset=errorlogx.offset;
425			errorlog.length=errorlogx.length;
426
427			/* Work out the offsets */
428
429			latbin=latlong_to_bin(errorlogx.latitude )-nodesx->latzero;
430			lonbin=latlong_to_bin(errorlogx.longitude)-nodesx->lonzero;
431			llbin=lonbin*nodesx->latbins+latbin;
432
433			for(;latlonbin<=llbin;latlonbin++)
434			offsets[latlonbin]=number_geo;
435
436			/* Write the data */
437
438			WriteFile(newfd,&errorlog,sizeof(ErrorLog));
439
440			number_geo++;
441			number++;
442
443			if(!(number%10000))
444			printf_middle("Writing Errors: Geographical=%"Pindex_t" Non-geographical=%"Pindex_t,number_geo,number_nongeo);
445			}
446
447			/* Write out the non-geographical errors */
448
449			SeekFile(oldfd,0);
450
451			while(!ReadFile(oldfd,&errorlogx,sizeof(ErrorLogX)))
452			{
453			ErrorLog errorlog={0};
454
455			if(errorlogx.latitude!=NO_LATLONG)
456			continue;
457
458			/* Create the ErrorLog */
459
460			errorlog.latoffset=0;
461			errorlog.lonoffset=0;
462
463			errorlog.offset=errorlogx.offset;
464			errorlog.length=errorlogx.length;
465
466			/* Write the data */
467
468			WriteFile(newfd,&errorlog,sizeof(ErrorLog));
469
470			number_nongeo++;
471			number++;
472
473			if(!(number%10000))
474			printf_middle("Writing Errors: Geographical=%"Pindex_t" Non-geographical=%"Pindex_t,number_geo,number_nongeo);
475			}
476
477			/* Close the input file */
478
479			CloseFile(oldfd);
480
481			DeleteFile(errorbinfilename);
482
483			/* Append the text from the log file */
484
485			size=SizeFile(errorlogfilename);
486
487			oldfd=ReOpenFile(errorlogfilename);
488
489			while(size)
490			{
491			char *buffer[4096];
492			off_t chunksize=(size>sizeof(buffer)?sizeof(buffer):size);
493
494			ReadFile(oldfd,buffer,chunksize);
495
496			WriteFile(newfd,buffer,chunksize);
497
498			size-=chunksize;
499			}
500
501			CloseFile(oldfd);
502
503			/* Finish off the offset indexing and write them out */
504
505			maxlatlonbins=nodesx->latbins*nodesx->lonbins;
506
507			for(;latlonbin<=maxlatlonbins;latlonbin++)
508			offsets[latlonbin]=number_geo;
509
510			SeekFile(newfd,sizeof(ErrorLogsFile));
511			WriteFile(newfd,offsets,(nodesx->latbinsnodesx->lonbins+1)sizeof(index_t));
512
513			free(offsets);
514
515			/* Write out the header structure */
516
517			errorlogsfile.number =number;
518			errorlogsfile.number_geo =number_geo;
519			errorlogsfile.number_nongeo=number_nongeo;
520
521			errorlogsfile.latbins=nodesx->latbins;
522			errorlogsfile.lonbins=nodesx->lonbins;
523
524			errorlogsfile.latzero=nodesx->latzero;
525			errorlogsfile.lonzero=nodesx->lonzero;
526
527			SeekFile(newfd,0);
528			WriteFile(newfd,&errorlogsfile,sizeof(ErrorLogsFile));
529
530			CloseFile(newfd);
531
532			/* Print the final message */
533
534			printf_last("Wrote Errors: Geographical=%"Pindex_t" Non-geographical=%"Pindex_t,number_geo,number_nongeo);
535			}