trunk/src/planetsplitter.c

/***************************************
 $Header: /home/amb/CVS/routino/src/planetsplitter.c,v 1.77 2010-09-15 17:41:22 amb Exp $

 OSM planet file splitter.

 Part of the Routino routing software.
 ******************/ /******************
 This file Copyright 2008-2010 Andrew M. Bishop

 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as published by
 the Free Software Foundation, either version 3 of the License, or
 (at your option) any later version.

 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU Affero General Public License for more details.

 You should have received a copy of the GNU Affero General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 ***************************************/


#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>

#include "types.h"
#include "ways.h"

#include "typesx.h"
#include "nodesx.h"
#include "segmentsx.h"
#include "waysx.h"
#include "superx.h"

#include "files.h"
#include "functions.h"
#include "functionsx.h"
#include "tagging.h"


/* Global variables */

/*+ The name of the temporary directory. +*/
char *option_tmpdirname=NULL;

/*+ The amount of RAM to use for filesorting. +*/
size_t option_filesort_ramsize=0;


/* Local functions */

static void print_usage(int detail);


/*++++++++++++++++++++++++++++++++++++++
  The main program for the planetsplitter.
  ++++++++++++++++++++++++++++++++++++++*/

int main(int argc,char** argv)
{
 NodesX    *Nodes;
 SegmentsX *Segments,*SuperSegments=NULL,*MergedSegments=NULL;
 WaysX     *Ways;
 int        iteration=0,quit=0;
 int        max_iterations=10;
 char      *dirname=NULL,*prefix=NULL,*tagging=NULL;
 int        option_parse_only=0,option_process_only=0;
 int        option_filenames=0;
 int        arg;

 /* Parse the command line arguments */

 for(arg=1;arg<argc;arg++)
   {
    if(!strcmp(argv[arg],"--help"))
       print_usage(1);
    else if(!strncmp(argv[arg],"--sort-ram-size=",16))
       option_filesort_ramsize=atoi(&argv[arg][16]);
    else if(!strncmp(argv[arg],"--dir=",6))
       dirname=&argv[arg][6];
    else if(!strncmp(argv[arg],"--tmpdir=",9))
       option_tmpdirname=&argv[arg][9];
    else if(!strncmp(argv[arg],"--prefix=",9))
       prefix=&argv[arg][9];
    else if(!strcmp(argv[arg],"--parse-only"))
       option_parse_only=1;
    else if(!strcmp(argv[arg],"--process-only"))
       option_process_only=1;
    else if(!strncmp(argv[arg],"--max-iterations=",17))
       max_iterations=atoi(&argv[arg][17]);
    else if(!strncmp(argv[arg],"--tagging=",10))
       tagging=&argv[arg][10];
    else if(argv[arg][0]=='-' && argv[arg][1]=='-')
       print_usage(0);
    else
       option_filenames++;
   }

 /* Check the specified command line options */

 if(option_parse_only && option_process_only)
    print_usage(0);

 if(option_filenames && option_process_only)
    print_usage(0);

 if(!option_filesort_ramsize)
   {
#if SLIM
       option_filesort_ramsize=64*1024*1024;
#else
       option_filesort_ramsize=256*1024*1024;
#endif
   }
 else
    option_filesort_ramsize*=1024*1024;

 if(!option_tmpdirname)
   {
    if(!dirname)
       option_tmpdirname=".";
    else
       option_tmpdirname=dirname;
   }

 if(tagging)
   {
    if(!ExistsFile(tagging))
      {
       fprintf(stderr,"Error: The '--tagging' option specifies a file that does not exist.\n");
       return(1);
      }
   }
 else
   {
    if(ExistsFile(FileName(dirname,prefix,"tagging.xml")))
       tagging=FileName(dirname,prefix,"tagging.xml");
    else if(ExistsFile(FileName(DATADIR,NULL,"tagging.xml")))
       tagging=FileName(DATADIR,NULL,"tagging.xml");
    else
      {
       fprintf(stderr,"Error: The '--tagging' option was not used and the default 'tagging.xml' does not exist.\n");
       return(1);
      }
   }

 if(ParseXMLTaggingRules(tagging))
   {
    fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging);
    return(1);
   }

 /* Create new node, segment and way variables */

 Nodes=NewNodeList(option_parse_only||option_process_only);

 Segments=NewSegmentList(option_parse_only||option_process_only);

 Ways=NewWayList(option_parse_only||option_process_only);

 /* Parse the file */

 if(option_filenames)
   {
    for(arg=1;arg<argc;arg++)
      {
       FILE *file;

       if(argv[arg][0]=='-' && argv[arg][1]=='-')
          continue;

       file=fopen(argv[arg],"rb");

       if(!file)
         {
          fprintf(stderr,"Cannot open file '%s' for reading [%s].\n",argv[arg],strerror(errno));
          exit(EXIT_FAILURE);
         }

       printf("\nParse OSM Data [%s]\n==============\n\n",argv[arg]);
       fflush(stdout);

       if(ParseOSM(file,Nodes,Segments,Ways))
          exit(EXIT_FAILURE);

       fclose(file);
      }
   }
 else if(!option_process_only)
   {
    printf("\nParse OSM Data\n==============\n\n");
    fflush(stdout);

    if(ParseOSM(stdin,Nodes,Segments,Ways))
       exit(EXIT_FAILURE);
   }

 if(option_parse_only)
   {
    FreeNodeList(Nodes,1);
    FreeSegmentList(Segments,1);
    FreeWayList(Ways,1);

    return(0);
   }

 /* Process the data */

 printf("\nProcess OSM Data\n================\n\n");
 fflush(stdout);

 /* Sort the nodes, segments and ways */

 SortNodeList(Nodes);

 SortSegmentList(Segments);

 SortWayList(Ways);

 /* Remove bad segments (must be after sorting the nodes and segments) */

 RemoveBadSegments(Nodes,Segments);

 /* Remove non-highway nodes (must be after removing the bad segments) */

 RemoveNonHighwayNodes(Nodes,Segments);

 /* Measure the segments and replace node/way id with index (must be after removing non-highway nodes) */

 UpdateSegments(Segments,Nodes,Ways);


 /* Repeated iteration on Super-Nodes and Super-Segments */

 do
   {
    printf("\nProcess Super-Data (iteration %d)\n================================%s\n\n",iteration,iteration>9?"=":"");
    fflush(stdout);

    if(iteration==0)
      {
       /* Select the super-nodes */

       ChooseSuperNodes(Nodes,Segments,Ways);

       /* Select the super-segments */

       SuperSegments=CreateSuperSegments(Nodes,Segments,Ways,iteration);
      }
    else
      {
       SegmentsX *SuperSegments2;

       /* Select the super-nodes */

       ChooseSuperNodes(Nodes,SuperSegments,Ways);

       /* Select the super-segments */

       SuperSegments2=CreateSuperSegments(Nodes,SuperSegments,Ways,iteration);

       if(SuperSegments->xnumber==SuperSegments2->xnumber)
          quit=1;

       FreeSegmentList(SuperSegments,0);

       SuperSegments=SuperSegments2;
      }

    /* Sort the super-segments */

    SortSegmentList(SuperSegments);

    /* Remove duplicated super-segments */

    DeduplicateSegments(SuperSegments,Nodes,Ways);

    iteration++;

    if(iteration>max_iterations)
       quit=1;
   }
 while(!quit);

 /* Combine the super-segments */

 printf("\nCombine Segments and Super-Segments\n===================================\n\n");
 fflush(stdout);

 /* Merge the super-segments */

 MergedSegments=MergeSuperSegments(Segments,SuperSegments);

 FreeSegmentList(Segments,0);

 FreeSegmentList(SuperSegments,0);

 Segments=MergedSegments;

 /* Rotate segments so that node1<node2 */

 RotateSegments(Segments);

 /* Sort the segments */

 SortSegmentList(Segments);

 /* Remove duplicated segments */

 DeduplicateSegments(Segments,Nodes,Ways);

 /* Cross reference the nodes and segments */

 printf("\nCross-Reference Nodes and Segments\n==================================\n\n");
 fflush(stdout);

 /* Sort the node list geographically */

 SortNodeListGeographically(Nodes);

 /* Create the real segments and nodes */

 CreateRealNodes(Nodes,iteration);

 CreateRealSegments(Segments,Ways);

 /* Fix the segment and node indexes */

 IndexNodes(Nodes,Segments);

 IndexSegments(Segments,Nodes);

 /* Output the results */

 printf("\nWrite Out Database Files\n========================\n\n");
 fflush(stdout);

 /* Write out the nodes */

 SaveNodeList(Nodes,FileName(dirname,prefix,"nodes.mem"));

 FreeNodeList(Nodes,0);

 /* Write out the segments */

 SaveSegmentList(Segments,FileName(dirname,prefix,"segments.mem"));

 FreeSegmentList(Segments,0);

 /* Write out the ways */

 SaveWayList(Ways,FileName(dirname,prefix,"ways.mem"));

 FreeWayList(Ways,0);

 return(0);
}


/*++++++++++++++++++++++++++++++++++++++
  Print out the usage information.

  int detail The level of detail to use - 0 = low, 1 = high.
  ++++++++++++++++++++++++++++++++++++++*/

static void print_usage(int detail)
{
 fprintf(stderr,
         "Usage: planetsplitter [--help]\n"
         "                      [--dir=<dirname>] [--prefix=<name>]\n"
         "                      [--sort-ram-size=<size>]\n"
         "                      [--tmpdir=<dirname>]\n"
         "                      [--parse-only | --process-only]\n"
         "                      [--max-iterations=<number>]\n"
         "                      [--tagging=<filename>]\n"
         "                      [<filename.osm> ...]\n");

 if(detail)
    fprintf(stderr,
            "\n"
            "--help                    Prints this information.\n"
            "\n"
            "--dir=<dirname>           The directory containing the routing database.\n"
            "--prefix=<name>           The filename prefix for the routing database.\n"
            "\n"
            "--sort-ram-size=<size>    The amount of RAM (in MB) to use for data sorting\n"
#if SLIM
            "                          (defaults to 64MB otherwise.)\n"
#else
            "                          (defaults to 256MB otherwise.)\n"
#endif
            "--tmpdir=<dirname>        The directory name for temporary files.\n"
            "                          (defaults to the '--dir' option directory.)\n"
            "\n"
            "--parse-only              Parse the input OSM files and store the results.\n"
            "--process-only            Process the stored results from previous option.\n"
            "\n"
            "--max-iterations=<number> The number of iterations for finding super-nodes.\n"
            "\n"
            "--tagging=<filename>      The name of the XML file containing the tagging rules\n"
            "                          (defaults to 'tagging.xml' with '--dir' and\n"
            "                           '--prefix' options or the file installed in\n"
            "                           '" DATADIR "').\n"
            "\n"
            "<filename.osm> ...        The name(s) of the file(s) to process (by default\n"
            "                          data is read from standard input).\n"
            "\n"
            "<transport> defaults to all but can be set to:\n"
            "%s"
            "\n"
            "<highway> can be selected from:\n"
            "%s"
            "\n"
            "<property> can be selected from:\n"
            "%s",
            TransportList(),HighwayList(),PropertyList());

 exit(!detail);
}
1	/***************************************
2	$Header: /home/amb/CVS/routino/src/planetsplitter.c,v 1.77 2010-09-15 17:41:22 amb Exp $
3
4	OSM planet file splitter.
5
6	Part of the Routino routing software.
7	****************/ /****************
8	This file Copyright 2008-2010 Andrew M. Bishop
9
10	This program is free software: you can redistribute it and/or modify
11	it under the terms of the GNU Affero General Public License as published by
12	the Free Software Foundation, either version 3 of the License, or
13	(at your option) any later version.
14
15	This program is distributed in the hope that it will be useful,
16	but WITHOUT ANY WARRANTY; without even the implied warranty of
17	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18	GNU Affero General Public License for more details.
19
20	You should have received a copy of the GNU Affero General Public License
21	along with this program. If not, see <http://www.gnu.org/licenses/>.
22	***************************************/
23
24
25	#include <stdio.h>
26	#include <stdlib.h>
27	#include <string.h>
28	#include <errno.h>
29
30	#include "types.h"
31	#include "ways.h"
32
33	#include "typesx.h"
34	#include "nodesx.h"
35	#include "segmentsx.h"
36	#include "waysx.h"
37	#include "superx.h"
38
39	#include "files.h"
40	#include "functions.h"
41	#include "functionsx.h"
42	#include "tagging.h"
43
44
45	/* Global variables */
46
47	/+ The name of the temporary directory. +/
48	char *option_tmpdirname=NULL;
49
50	/+ The amount of RAM to use for filesorting. +/
51	size_t option_filesort_ramsize=0;
52
53
54	/* Local functions */
55
56	static void print_usage(int detail);
57
58
59	/*++++++++++++++++++++++++++++++++++++++
60	The main program for the planetsplitter.
61	++++++++++++++++++++++++++++++++++++++*/
62
63	int main(int argc,char** argv)
64	{
65	NodesX *Nodes;
66	SegmentsX Segments,SuperSegments=NULL,*MergedSegments=NULL;
67	WaysX *Ways;
68	int iteration=0,quit=0;
69	int max_iterations=10;
70	char dirname=NULL,prefix=NULL,*tagging=NULL;
71	int option_parse_only=0,option_process_only=0;
72	int option_filenames=0;
73	int arg;
74
75	/* Parse the command line arguments */
76
77	for(arg=1;arg<argc;arg++)
78	{
79	if(!strcmp(argv[arg],"--help"))
80	print_usage(1);
81	else if(!strncmp(argv[arg],"--sort-ram-size=",16))
82	option_filesort_ramsize=atoi(&argv[arg][16]);
83	else if(!strncmp(argv[arg],"--dir=",6))
84	dirname=&argv[arg][6];
85	else if(!strncmp(argv[arg],"--tmpdir=",9))
86	option_tmpdirname=&argv[arg][9];
87	else if(!strncmp(argv[arg],"--prefix=",9))
88	prefix=&argv[arg][9];
89	else if(!strcmp(argv[arg],"--parse-only"))
90	option_parse_only=1;
91	else if(!strcmp(argv[arg],"--process-only"))
92	option_process_only=1;
93	else if(!strncmp(argv[arg],"--max-iterations=",17))
94	max_iterations=atoi(&argv[arg][17]);
95	else if(!strncmp(argv[arg],"--tagging=",10))
96	tagging=&argv[arg][10];
97	else if(argv[arg][0]=='-' && argv[arg][1]=='-')
98	print_usage(0);
99	else
100	option_filenames++;
101	}
102
103	/* Check the specified command line options */
104
105	if(option_parse_only && option_process_only)
106	print_usage(0);
107
108	if(option_filenames && option_process_only)
109	print_usage(0);
110
111	if(!option_filesort_ramsize)
112	{
113	#if SLIM
114	option_filesort_ramsize=6410241024;
115	#else
116	option_filesort_ramsize=25610241024;
117	#endif
118	}
119	else
120	option_filesort_ramsize=10241024;
121
122	if(!option_tmpdirname)
123	{
124	if(!dirname)
125	option_tmpdirname=".";
126	else
127	option_tmpdirname=dirname;
128	}
129
130	if(tagging)
131	{
132	if(!ExistsFile(tagging))
133	{
134	fprintf(stderr,"Error: The '--tagging' option specifies a file that does not exist.\n");
135	return(1);
136	}
137	}
138	else
139	{
140	if(ExistsFile(FileName(dirname,prefix,"tagging.xml")))
141	tagging=FileName(dirname,prefix,"tagging.xml");
142	else if(ExistsFile(FileName(DATADIR,NULL,"tagging.xml")))
143	tagging=FileName(DATADIR,NULL,"tagging.xml");
144	else
145	{
146	fprintf(stderr,"Error: The '--tagging' option was not used and the default 'tagging.xml' does not exist.\n");
147	return(1);
148	}
149	}
150
151	if(ParseXMLTaggingRules(tagging))
152	{
153	fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging);
154	return(1);
155	}
156
157	/* Create new node, segment and way variables */
158
159	Nodes=NewNodeList(option_parse_only\|\|option_process_only);
160
161	Segments=NewSegmentList(option_parse_only\|\|option_process_only);
162
163	Ways=NewWayList(option_parse_only\|\|option_process_only);
164
165	/* Parse the file */
166
167	if(option_filenames)
168	{
169	for(arg=1;arg<argc;arg++)
170	{
171	FILE *file;
172
173	if(argv[arg][0]=='-' && argv[arg][1]=='-')
174	continue;
175
176	file=fopen(argv[arg],"rb");
177
178	if(!file)
179	{
180	fprintf(stderr,"Cannot open file '%s' for reading [%s].\n",argv[arg],strerror(errno));
181	exit(EXIT_FAILURE);
182	}
183
184	printf("\nParse OSM Data [%s]\n==============\n\n",argv[arg]);
185	fflush(stdout);
186
187	if(ParseOSM(file,Nodes,Segments,Ways))
188	exit(EXIT_FAILURE);
189
190	fclose(file);
191	}
192	}
193	else if(!option_process_only)
194	{
195	printf("\nParse OSM Data\n==============\n\n");
196	fflush(stdout);
197
198	if(ParseOSM(stdin,Nodes,Segments,Ways))
199	exit(EXIT_FAILURE);
200	}
201
202	if(option_parse_only)
203	{
204	FreeNodeList(Nodes,1);
205	FreeSegmentList(Segments,1);
206	FreeWayList(Ways,1);
207
208	return(0);
209	}
210
211	/* Process the data */
212
213	printf("\nProcess OSM Data\n================\n\n");
214	fflush(stdout);
215
216	/* Sort the nodes, segments and ways */
217
218	SortNodeList(Nodes);
219
220	SortSegmentList(Segments);
221
222	SortWayList(Ways);
223
224	/* Remove bad segments (must be after sorting the nodes and segments) */
225
226	RemoveBadSegments(Nodes,Segments);
227
228	/* Remove non-highway nodes (must be after removing the bad segments) */
229
230	RemoveNonHighwayNodes(Nodes,Segments);
231
232	/* Measure the segments and replace node/way id with index (must be after removing non-highway nodes) */
233
234	UpdateSegments(Segments,Nodes,Ways);
235
236
237	/* Repeated iteration on Super-Nodes and Super-Segments */
238
239	do
240	{
241	printf("\nProcess Super-Data (iteration %d)\n================================%s\n\n",iteration,iteration>9?"=":"");
242	fflush(stdout);
243
244	if(iteration==0)
245	{
246	/* Select the super-nodes */
247
248	ChooseSuperNodes(Nodes,Segments,Ways);
249
250	/* Select the super-segments */
251
252	SuperSegments=CreateSuperSegments(Nodes,Segments,Ways,iteration);
253	}
254	else
255	{
256	SegmentsX *SuperSegments2;
257
258	/* Select the super-nodes */
259
260	ChooseSuperNodes(Nodes,SuperSegments,Ways);
261
262	/* Select the super-segments */
263
264	SuperSegments2=CreateSuperSegments(Nodes,SuperSegments,Ways,iteration);
265
266	if(SuperSegments->xnumber==SuperSegments2->xnumber)
267	quit=1;
268
269	FreeSegmentList(SuperSegments,0);
270
271	SuperSegments=SuperSegments2;
272	}
273
274	/* Sort the super-segments */
275
276	SortSegmentList(SuperSegments);
277
278	/* Remove duplicated super-segments */
279
280	DeduplicateSegments(SuperSegments,Nodes,Ways);
281
282	iteration++;
283
284	if(iteration>max_iterations)
285	quit=1;
286	}
287	while(!quit);
288
289	/* Combine the super-segments */
290
291	printf("\nCombine Segments and Super-Segments\n===================================\n\n");
292	fflush(stdout);
293
294	/* Merge the super-segments */
295
296	MergedSegments=MergeSuperSegments(Segments,SuperSegments);
297
298	FreeSegmentList(Segments,0);
299
300	FreeSegmentList(SuperSegments,0);
301
302	Segments=MergedSegments;
303
304	/* Rotate segments so that node1<node2 */
305
306	RotateSegments(Segments);
307
308	/* Sort the segments */
309
310	SortSegmentList(Segments);
311
312	/* Remove duplicated segments */
313
314	DeduplicateSegments(Segments,Nodes,Ways);
315
316	/* Cross reference the nodes and segments */
317
318	printf("\nCross-Reference Nodes and Segments\n==================================\n\n");
319	fflush(stdout);
320
321	/* Sort the node list geographically */
322
323	SortNodeListGeographically(Nodes);
324
325	/* Create the real segments and nodes */
326
327	CreateRealNodes(Nodes,iteration);
328
329	CreateRealSegments(Segments,Ways);
330
331	/* Fix the segment and node indexes */
332
333	IndexNodes(Nodes,Segments);
334
335	IndexSegments(Segments,Nodes);
336
337	/* Output the results */
338
339	printf("\nWrite Out Database Files\n========================\n\n");
340	fflush(stdout);
341
342	/* Write out the nodes */
343
344	SaveNodeList(Nodes,FileName(dirname,prefix,"nodes.mem"));
345
346	FreeNodeList(Nodes,0);
347
348	/* Write out the segments */
349
350	SaveSegmentList(Segments,FileName(dirname,prefix,"segments.mem"));
351
352	FreeSegmentList(Segments,0);
353
354	/* Write out the ways */
355
356	SaveWayList(Ways,FileName(dirname,prefix,"ways.mem"));
357
358	FreeWayList(Ways,0);
359
360	return(0);
361	}
362
363
364	/*++++++++++++++++++++++++++++++++++++++
365	Print out the usage information.
366
367	int detail The level of detail to use - 0 = low, 1 = high.
368	++++++++++++++++++++++++++++++++++++++*/
369
370	static void print_usage(int detail)
371	{
372	fprintf(stderr,
373	"Usage: planetsplitter [--help]\n"
374	" [--dir=<dirname>] [--prefix=<name>]\n"
375	" [--sort-ram-size=<size>]\n"
376	" [--tmpdir=<dirname>]\n"
377	" [--parse-only \| --process-only]\n"
378	" [--max-iterations=<number>]\n"
379	" [--tagging=<filename>]\n"
380	" [<filename.osm> ...]\n");
381
382	if(detail)
383	fprintf(stderr,
384	"\n"
385	"--help Prints this information.\n"
386	"\n"
387	"--dir=<dirname> The directory containing the routing database.\n"
388	"--prefix=<name> The filename prefix for the routing database.\n"
389	"\n"
390	"--sort-ram-size=<size> The amount of RAM (in MB) to use for data sorting\n"
391	#if SLIM
392	" (defaults to 64MB otherwise.)\n"
393	#else
394	" (defaults to 256MB otherwise.)\n"
395	#endif
396	"--tmpdir=<dirname> The directory name for temporary files.\n"
397	" (defaults to the '--dir' option directory.)\n"
398	"\n"
399	"--parse-only Parse the input OSM files and store the results.\n"
400	"--process-only Process the stored results from previous option.\n"
401	"\n"
402	"--max-iterations=<number> The number of iterations for finding super-nodes.\n"
403	"\n"
404	"--tagging=<filename> The name of the XML file containing the tagging rules\n"
405	" (defaults to 'tagging.xml' with '--dir' and\n"
406	" '--prefix' options or the file installed in\n"
407	" '" DATADIR "').\n"
408	"\n"
409	"<filename.osm> ... The name(s) of the file(s) to process (by default\n"
410	" data is read from standard input).\n"
411	"\n"
412	"<transport> defaults to all but can be set to:\n"
413	"%s"
414	"\n"
415	"<highway> can be selected from:\n"
416	"%s"
417	"\n"
418	"<property> can be selected from:\n"
419	"%s",
420	TransportList(),HighwayList(),PropertyList());
421
422	exit(!detail);
423	}