extras/find-fixme/fixme-finder.c

/***************************************
 OSM planet file fixme finder.

 Part of the Routino routing software.
 ******************/ /******************
 This file Copyright 2008-2015, 2023 Andrew M. Bishop

 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as published by
 the Free Software Foundation, either version 3 of the License, or
 (at your option) any later version.

 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU Affero General Public License for more details.

 You should have received a copy of the GNU Affero General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 ***************************************/


#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>

#include "version.h"

#include "types.h"
#include "ways.h"

#include "typesx.h"
#include "nodesx.h"
#include "waysx.h"
#include "relationsx.h"

#include "files.h"
#include "logging.h"
#include "errorlogx.h"
#include "functions.h"
#include "osmparser.h"
#include "tagging.h"
#include "uncompress.h"


/* Global variables */

/*+ The name of the temporary directory. +*/
char *option_tmpdirname=NULL;

/*+ The amount of RAM to use for filesorting. +*/
size_t option_filesort_ramsize=0;

/*+ The number of threads to use for filesorting. +*/
int option_filesort_threads=1;


/* Local functions */

static void print_usage(int detail,const char *argerr,const char *err);


/*++++++++++++++++++++++++++++++++++++++
  The main program for the find-fixme.
  ++++++++++++++++++++++++++++++++++++++*/

int main(int argc,char** argv)
{
 NodesX     *OSMNodes;
 WaysX      *OSMWays;
 RelationsX *OSMRelations;
 ErrorLogsX *OSMErrorLogs;
 char       *dirname=NULL,*prefix=NULL,*tagging=NULL;
 int         option_keep=1;
 int         option_filenames=0;
 int         arg;

 printf_program_start();

 /* Parse the command line arguments */

 for(arg=1;arg<argc;arg++)
   {
    if(!strcmp(argv[arg],"--version"))
       print_usage(-1,NULL,NULL);
    else if(!strcmp(argv[arg],"--help"))
       print_usage(1,NULL,NULL);
    else if(!strncmp(argv[arg],"--dir=",6))
       dirname=&argv[arg][6];
    else if(!strncmp(argv[arg],"--sort-ram-size=",16))
       option_filesort_ramsize=atoi(&argv[arg][16]);
#if defined(USE_PTHREADS) && USE_PTHREADS
    else if(!strncmp(argv[arg],"--sort-threads=",15))
       option_filesort_threads=atoi(&argv[arg][15]);
#endif
    else if(!strncmp(argv[arg],"--tmpdir=",9))
       option_tmpdirname=&argv[arg][9];
    else if(!strncmp(argv[arg],"--tagging=",10))
       tagging=&argv[arg][10];
    else if(!strcmp(argv[arg],"--loggable"))
       option_loggable=1;
    else if(!strcmp(argv[arg],"--logtime"))
       option_logtime=1;
    else if(!strcmp(argv[arg],"--logmemory"))
       option_logmemory=1;
    else if(argv[arg][0]=='-' && argv[arg][1]=='-')
       print_usage(0,argv[arg],NULL);
    else
       option_filenames++;
   }

 /* Check the specified command line options */

 if(!option_filesort_ramsize)
   {
#if SLIM
    option_filesort_ramsize=256*1024*1024;
#else
    option_filesort_ramsize=1024*1024*1024;
#endif
   }
 else
    option_filesort_ramsize*=1024*1024;

#if defined(USE_PTHREADS) && USE_PTHREADS
 if(option_filesort_threads<1 || option_filesort_threads>32)
    print_usage(0,NULL,"Sorting threads '--sort-threads=...' must be small positive integer.");
#endif

 if(!option_tmpdirname)
   {
    if(!dirname)
       option_tmpdirname=".";
    else
       option_tmpdirname=dirname;
   }

 if(tagging)
   {
    if(!ExistsFile(tagging))
      {
       fprintf(stderr,"Error: The '--tagging' option specifies a file that does not exist.\n");
       exit(EXIT_FAILURE);
      }
   }
 else
   {
    tagging=FileName(dirname,prefix,"fixme.xml");

    if(!ExistsFile(tagging))
      {
       fprintf(stderr,"Error: The '--tagging' option was not used and the default 'fixme.xml' does not exist.\n");
       exit(EXIT_FAILURE);
      }
   }

 if(ParseXMLTaggingRules(tagging))
   {
    fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging);
    exit(EXIT_FAILURE);
   }

 /* Create new node, segment, way and relation variables */

 OSMNodes=NewNodeList(0,0);

 OSMWays=NewWayList(0,0);

 OSMRelations=NewRelationList(0,0);

 /* Create the error log file */

 open_errorlog(FileName(dirname,prefix,"fixme.log"),0,option_keep);

 /* Parse the file */

 for(arg=1;arg<argc;arg++)
   {
    int fd;
    char *filename,*p;

    if(argv[arg][0]=='-' && argv[arg][1]=='-')
       continue;

    filename=strcpy(malloc(strlen(argv[arg])+1),argv[arg]);

    fd=OpenFile(filename);

    if((p=strstr(filename,".bz2")) && !strcmp(p,".bz2"))
      {
       fd=Uncompress_Bzip2(fd);
       *p=0;
      }

    if((p=strstr(filename,".gz")) && !strcmp(p,".gz"))
      {
       fd=Uncompress_Gzip(fd);
       *p=0;
      }

    if((p=strstr(filename,".xz")) && !strcmp(p,".xz"))
      {
       fd=Uncompress_Xz(fd);
       *p=0;
      }

    printf("\nParse OSM Data [%s]\n==============\n\n",filename);
    fflush(stdout);

    if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf"))
      {
       if(ParsePBFFile(fd,OSMNodes,OSMWays,OSMRelations))
          exit(EXIT_FAILURE);
      }
    else if((p=strstr(filename,".o5m")) && !strcmp(p,".o5m"))
      {
       if(ParseO5MFile(fd,OSMNodes,OSMWays,OSMRelations))
          exit(EXIT_FAILURE);
      }
    else
      {
       if(ParseOSMFile(fd,OSMNodes,OSMWays,OSMRelations))
          exit(EXIT_FAILURE);
      }

    CloseFile(fd);

    free(filename);
   }

 DeleteXMLTaggingRules();

 FinishNodeList(OSMNodes);
 FinishWayList(OSMWays);
 FinishRelationList(OSMRelations);

 /* Sort the data */

 printf("\nSort OSM Data\n=============\n\n");
 fflush(stdout);

 /* Sort the nodes, ways and relations */

 SortNodeList(OSMNodes);

 SortWayList(OSMWays);

 SortRelationList(OSMRelations);

 /* Process the data */

 RenameFile(OSMNodes->filename_tmp,OSMNodes->filename);
 RenameFile(OSMWays->filename_tmp,OSMWays->filename);
 RenameFile(OSMRelations->rrfilename_tmp,OSMRelations->rrfilename);
 RenameFile(OSMRelations->trfilename_tmp,OSMRelations->trfilename);

 close_errorlog();

 printf("\nCreate Error Log\n================\n\n");
 fflush(stdout);

 OSMErrorLogs=NewErrorLogList();

 ProcessErrorLogs(OSMErrorLogs,OSMNodes,OSMWays,OSMRelations);

 SortErrorLogsGeographically(OSMErrorLogs);

 SaveErrorLogs(OSMErrorLogs,FileName(dirname,prefix,"fixme.mem"));

 FreeErrorLogList(OSMErrorLogs);

 /* Free the memory (delete the temporary files) */

 FreeNodeList(OSMNodes,0);
 FreeWayList(OSMWays,0);
 FreeRelationList(OSMRelations,0);

 printf("\n");
 fflush(stdout);

 printf_program_end();

 exit(EXIT_SUCCESS);
}


/*++++++++++++++++++++++++++++++++++++++
  Print out the usage information.

  int detail The level of detail to use: -1 = just version number, 0 = low detail, 1 = full details.

  const char *argerr The argument that gave the error (if there is one).

  const char *err Other error message (if there is one).
  ++++++++++++++++++++++++++++++++++++++*/

static void print_usage(int detail,const char *argerr,const char *err)
{
 if(detail<0)
   {
    fprintf(stderr,
            "Routino version " ROUTINO_VERSION " " ROUTINO_URL ".\n"
            );
   }

 if(detail>=0)
   {
    fprintf(stderr,
            "Usage: fixme-finder [--version]\n"
            "                    [--help]\n"
            "                    [--dir=<dirname>]\n"
#if defined(USE_PTHREADS) && USE_PTHREADS
            "                    [--sort-ram-size=<size>] [--sort-threads=<number>]\n"
#else
            "                    [--sort-ram-size=<size>]\n"
#endif
            "                    [--tmpdir=<dirname>]\n"
            "                    [--tagging=<filename>]\n"
            "                    [--loggable] [--logtime] [--logmemory]\n"
            "                    [<filename.osm> ...\n"
            "                     | <filename.pbf> ...\n"
            "                     | <filename.o5m> ..."
#if defined(USE_BZIP2) && USE_BZIP2
            "\n                     | <filename.(osm|o5m).bz2> ..."
#endif
#if defined(USE_GZIP) && USE_GZIP
            "\n                     | <filename.(osm|o5m).gz> ..."
#endif
#if defined(USE_XZ) && USE_XZ
            "\n                     | <filename.(osm|o5m).xz> ..."
#endif
            "]\n");

    if(argerr)
       fprintf(stderr,
               "\n"
               "Error with command line parameter: %s\n",argerr);

    if(err)
       fprintf(stderr,
               "\n"
               "Error: %s\n",err);
   }

 if(detail==1)
    fprintf(stderr,
            "\n"
            "--version                 Print the version of Routino.\n"
            "\n"
            "--help                    Prints this information.\n"
            "\n"
            "--dir=<dirname>           The directory containing the fixme database.\n"
            "\n"
            "--sort-ram-size=<size>    The amount of RAM (in MB) to use for data sorting\n"
#if SLIM
            "                          (defaults to 256MB otherwise.)\n"
#else
            "                          (defaults to 1024MB otherwise.)\n"
#endif
#if defined(USE_PTHREADS) && USE_PTHREADS
            "--sort-threads=<number>   The number of threads to use for data sorting.\n"
#endif
            "\n"
            "--tmpdir=<dirname>        The directory name for temporary files.\n"
            "                          (defaults to the '--dir' option directory.)\n"
            "\n"
            "--tagging=<filename>      The name of the XML file containing the tagging rules\n"
            "                          (defaults to 'fixme.xml' with '--dir' option)\n"
            "\n"
            "--loggable                Print progress messages suitable for logging to file.\n"
            "--logtime                 Print the elapsed time for each processing step.\n"
            "--logmemory               Print the max allocated/mapped memory for each step.\n"
            "\n"
            "<filename.osm>, <filename.pbf>, <filename.o5m>\n"
            "                          The name(s) of the file(s) to read and parse.\n"
            "                          Filenames ending '.pbf' read as PBF, filenames ending\n"
            "                          '.o5m' read as O5M, others as XML.\n"
#if defined(USE_BZIP2) && USE_BZIP2
            "                          Filenames ending '.bz2' will be bzip2 uncompressed.\n"
#endif
#if defined(USE_GZIP) && USE_GZIP
            "                          Filenames ending '.gz' will be gzip uncompressed.\n"
#endif
#if defined(USE_XZ) && USE_XZ
            "                          Filenames ending '.xz' will be xz uncompressed.\n"
#endif
            );

 exit(!detail);
}
1	/***************************************
2	OSM planet file fixme finder.
3
4	Part of the Routino routing software.
5	****************/ /****************
6	This file Copyright 2008-2015, 2023 Andrew M. Bishop
7
8	This program is free software: you can redistribute it and/or modify
9	it under the terms of the GNU Affero General Public License as published by
10	the Free Software Foundation, either version 3 of the License, or
11	(at your option) any later version.
12
13	This program is distributed in the hope that it will be useful,
14	but WITHOUT ANY WARRANTY; without even the implied warranty of
15	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16	GNU Affero General Public License for more details.
17
18	You should have received a copy of the GNU Affero General Public License
19	along with this program. If not, see <http://www.gnu.org/licenses/>.
20	***************************************/
21
22
23	#include <stdio.h>
24	#include <stdlib.h>
25	#include <string.h>
26	#include <errno.h>
27
28	#include "version.h"
29
30	#include "types.h"
31	#include "ways.h"
32
33	#include "typesx.h"
34	#include "nodesx.h"
35	#include "waysx.h"
36	#include "relationsx.h"
37
38	#include "files.h"
39	#include "logging.h"
40	#include "errorlogx.h"
41	#include "functions.h"
42	#include "osmparser.h"
43	#include "tagging.h"
44	#include "uncompress.h"
45
46
47	/* Global variables */
48
49	/+ The name of the temporary directory. +/
50	char *option_tmpdirname=NULL;
51
52	/+ The amount of RAM to use for filesorting. +/
53	size_t option_filesort_ramsize=0;
54
55	/+ The number of threads to use for filesorting. +/
56	int option_filesort_threads=1;
57
58
59	/* Local functions */
60
61	static void print_usage(int detail,const char argerr,const char err);
62
63
64	/*++++++++++++++++++++++++++++++++++++++
65	The main program for the find-fixme.
66	++++++++++++++++++++++++++++++++++++++*/
67
68	int main(int argc,char** argv)
69	{
70	NodesX *OSMNodes;
71	WaysX *OSMWays;
72	RelationsX *OSMRelations;
73	ErrorLogsX *OSMErrorLogs;
74	char dirname=NULL,prefix=NULL,*tagging=NULL;
75	int option_keep=1;
76	int option_filenames=0;
77	int arg;
78
79	printf_program_start();
80
81	/* Parse the command line arguments */
82
83	for(arg=1;arg<argc;arg++)
84	{
85	if(!strcmp(argv[arg],"--version"))
86	print_usage(-1,NULL,NULL);
87	else if(!strcmp(argv[arg],"--help"))
88	print_usage(1,NULL,NULL);
89	else if(!strncmp(argv[arg],"--dir=",6))
90	dirname=&argv[arg][6];
91	else if(!strncmp(argv[arg],"--sort-ram-size=",16))
92	option_filesort_ramsize=atoi(&argv[arg][16]);
93	#if defined(USE_PTHREADS) && USE_PTHREADS
94	else if(!strncmp(argv[arg],"--sort-threads=",15))
95	option_filesort_threads=atoi(&argv[arg][15]);
96	#endif
97	else if(!strncmp(argv[arg],"--tmpdir=",9))
98	option_tmpdirname=&argv[arg][9];
99	else if(!strncmp(argv[arg],"--tagging=",10))
100	tagging=&argv[arg][10];
101	else if(!strcmp(argv[arg],"--loggable"))
102	option_loggable=1;
103	else if(!strcmp(argv[arg],"--logtime"))
104	option_logtime=1;
105	else if(!strcmp(argv[arg],"--logmemory"))
106	option_logmemory=1;
107	else if(argv[arg][0]=='-' && argv[arg][1]=='-')
108	print_usage(0,argv[arg],NULL);
109	else
110	option_filenames++;
111	}
112
113	/* Check the specified command line options */
114
115	if(!option_filesort_ramsize)
116	{
117	#if SLIM
118	option_filesort_ramsize=25610241024;
119	#else
120	option_filesort_ramsize=102410241024;
121	#endif
122	}
123	else
124	option_filesort_ramsize=10241024;
125
126	#if defined(USE_PTHREADS) && USE_PTHREADS
127	if(option_filesort_threads<1 \|\| option_filesort_threads>32)
128	print_usage(0,NULL,"Sorting threads '--sort-threads=...' must be small positive integer.");
129	#endif
130
131	if(!option_tmpdirname)
132	{
133	if(!dirname)
134	option_tmpdirname=".";
135	else
136	option_tmpdirname=dirname;
137	}
138
139	if(tagging)
140	{
141	if(!ExistsFile(tagging))
142	{
143	fprintf(stderr,"Error: The '--tagging' option specifies a file that does not exist.\n");
144	exit(EXIT_FAILURE);
145	}
146	}
147	else
148	{
149	tagging=FileName(dirname,prefix,"fixme.xml");
150
151	if(!ExistsFile(tagging))
152	{
153	fprintf(stderr,"Error: The '--tagging' option was not used and the default 'fixme.xml' does not exist.\n");
154	exit(EXIT_FAILURE);
155	}
156	}
157
158	if(ParseXMLTaggingRules(tagging))
159	{
160	fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging);
161	exit(EXIT_FAILURE);
162	}
163
164	/* Create new node, segment, way and relation variables */
165
166	OSMNodes=NewNodeList(0,0);
167
168	OSMWays=NewWayList(0,0);
169
170	OSMRelations=NewRelationList(0,0);
171
172	/* Create the error log file */
173
174	open_errorlog(FileName(dirname,prefix,"fixme.log"),0,option_keep);
175
176	/* Parse the file */
177
178	for(arg=1;arg<argc;arg++)
179	{
180	int fd;
181	char filename,p;
182
183	if(argv[arg][0]=='-' && argv[arg][1]=='-')
184	continue;
185
186	filename=strcpy(malloc(strlen(argv[arg])+1),argv[arg]);
187
188	fd=OpenFile(filename);
189
190	if((p=strstr(filename,".bz2")) && !strcmp(p,".bz2"))
191	{
192	fd=Uncompress_Bzip2(fd);
193	*p=0;
194	}
195
196	if((p=strstr(filename,".gz")) && !strcmp(p,".gz"))
197	{
198	fd=Uncompress_Gzip(fd);
199	*p=0;
200	}
201
202	if((p=strstr(filename,".xz")) && !strcmp(p,".xz"))
203	{
204	fd=Uncompress_Xz(fd);
205	*p=0;
206	}
207
208	printf("\nParse OSM Data [%s]\n==============\n\n",filename);
209	fflush(stdout);
210
211	if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf"))
212	{
213	if(ParsePBFFile(fd,OSMNodes,OSMWays,OSMRelations))
214	exit(EXIT_FAILURE);
215	}
216	else if((p=strstr(filename,".o5m")) && !strcmp(p,".o5m"))
217	{
218	if(ParseO5MFile(fd,OSMNodes,OSMWays,OSMRelations))
219	exit(EXIT_FAILURE);
220	}
221	else
222	{
223	if(ParseOSMFile(fd,OSMNodes,OSMWays,OSMRelations))
224	exit(EXIT_FAILURE);
225	}
226
227	CloseFile(fd);
228
229	free(filename);
230	}
231
232	DeleteXMLTaggingRules();
233
234	FinishNodeList(OSMNodes);
235	FinishWayList(OSMWays);
236	FinishRelationList(OSMRelations);
237
238	/* Sort the data */
239
240	printf("\nSort OSM Data\n=============\n\n");
241	fflush(stdout);
242
243	/* Sort the nodes, ways and relations */
244
245	SortNodeList(OSMNodes);
246
247	SortWayList(OSMWays);
248
249	SortRelationList(OSMRelations);
250
251	/* Process the data */
252
253	RenameFile(OSMNodes->filename_tmp,OSMNodes->filename);
254	RenameFile(OSMWays->filename_tmp,OSMWays->filename);
255	RenameFile(OSMRelations->rrfilename_tmp,OSMRelations->rrfilename);
256	RenameFile(OSMRelations->trfilename_tmp,OSMRelations->trfilename);
257
258	close_errorlog();
259
260	printf("\nCreate Error Log\n================\n\n");
261	fflush(stdout);
262
263	OSMErrorLogs=NewErrorLogList();
264
265	ProcessErrorLogs(OSMErrorLogs,OSMNodes,OSMWays,OSMRelations);
266
267	SortErrorLogsGeographically(OSMErrorLogs);
268
269	SaveErrorLogs(OSMErrorLogs,FileName(dirname,prefix,"fixme.mem"));
270
271	FreeErrorLogList(OSMErrorLogs);
272
273	/* Free the memory (delete the temporary files) */
274
275	FreeNodeList(OSMNodes,0);
276	FreeWayList(OSMWays,0);
277	FreeRelationList(OSMRelations,0);
278
279	printf("\n");
280	fflush(stdout);
281
282	printf_program_end();
283
284	exit(EXIT_SUCCESS);
285	}
286
287
288	/*++++++++++++++++++++++++++++++++++++++
289	Print out the usage information.
290
291	int detail The level of detail to use: -1 = just version number, 0 = low detail, 1 = full details.
292
293	const char *argerr The argument that gave the error (if there is one).
294
295	const char *err Other error message (if there is one).
296	++++++++++++++++++++++++++++++++++++++*/
297
298	static void print_usage(int detail,const char argerr,const char err)
299	{
300	if(detail<0)
301	{
302	fprintf(stderr,
303	"Routino version " ROUTINO_VERSION " " ROUTINO_URL ".\n"
304	);
305	}
306
307	if(detail>=0)
308	{
309	fprintf(stderr,
310	"Usage: fixme-finder [--version]\n"
311	" [--help]\n"
312	" [--dir=<dirname>]\n"
313	#if defined(USE_PTHREADS) && USE_PTHREADS
314	" [--sort-ram-size=<size>] [--sort-threads=<number>]\n"
315	#else
316	" [--sort-ram-size=<size>]\n"
317	#endif
318	" [--tmpdir=<dirname>]\n"
319	" [--tagging=<filename>]\n"
320	" [--loggable] [--logtime] [--logmemory]\n"
321	" [<filename.osm> ...\n"
322	" \| <filename.pbf> ...\n"
323	" \| <filename.o5m> ..."
324	#if defined(USE_BZIP2) && USE_BZIP2
325	"\n \| <filename.(osm\|o5m).bz2> ..."
326	#endif
327	#if defined(USE_GZIP) && USE_GZIP
328	"\n \| <filename.(osm\|o5m).gz> ..."
329	#endif
330	#if defined(USE_XZ) && USE_XZ
331	"\n \| <filename.(osm\|o5m).xz> ..."
332	#endif
333	"]\n");
334
335	if(argerr)
336	fprintf(stderr,
337	"\n"
338	"Error with command line parameter: %s\n",argerr);
339
340	if(err)
341	fprintf(stderr,
342	"\n"
343	"Error: %s\n",err);
344	}
345
346	if(detail==1)
347	fprintf(stderr,
348	"\n"
349	"--version Print the version of Routino.\n"
350	"\n"
351	"--help Prints this information.\n"
352	"\n"
353	"--dir=<dirname> The directory containing the fixme database.\n"
354	"\n"
355	"--sort-ram-size=<size> The amount of RAM (in MB) to use for data sorting\n"
356	#if SLIM
357	" (defaults to 256MB otherwise.)\n"
358	#else
359	" (defaults to 1024MB otherwise.)\n"
360	#endif
361	#if defined(USE_PTHREADS) && USE_PTHREADS
362	"--sort-threads=<number> The number of threads to use for data sorting.\n"
363	#endif
364	"\n"
365	"--tmpdir=<dirname> The directory name for temporary files.\n"
366	" (defaults to the '--dir' option directory.)\n"
367	"\n"
368	"--tagging=<filename> The name of the XML file containing the tagging rules\n"
369	" (defaults to 'fixme.xml' with '--dir' option)\n"
370	"\n"
371	"--loggable Print progress messages suitable for logging to file.\n"
372	"--logtime Print the elapsed time for each processing step.\n"
373	"--logmemory Print the max allocated/mapped memory for each step.\n"
374	"\n"
375	"<filename.osm>, <filename.pbf>, <filename.o5m>\n"
376	" The name(s) of the file(s) to read and parse.\n"
377	" Filenames ending '.pbf' read as PBF, filenames ending\n"
378	" '.o5m' read as O5M, others as XML.\n"
379	#if defined(USE_BZIP2) && USE_BZIP2
380	" Filenames ending '.bz2' will be bzip2 uncompressed.\n"
381	#endif
382	#if defined(USE_GZIP) && USE_GZIP
383	" Filenames ending '.gz' will be gzip uncompressed.\n"
384	#endif
385	#if defined(USE_XZ) && USE_XZ
386	" Filenames ending '.xz' will be xz uncompressed.\n"
387	#endif
388	);
389
390	exit(!detail);
391	}