trunk/src/planetsplitter.c

/***************************************
 OSM planet file splitter.

 Part of the Routino routing software.
 ******************/ /******************
 This file Copyright 2008-2015 Andrew M. Bishop

 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as published by
 the Free Software Foundation, either version 3 of the License, or
 (at your option) any later version.

 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU Affero General Public License for more details.

 You should have received a copy of the GNU Affero General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 ***************************************/


#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>

#include "version.h"

#include "types.h"
#include "ways.h"

#include "typesx.h"
#include "nodesx.h"
#include "segmentsx.h"
#include "waysx.h"
#include "relationsx.h"
#include "superx.h"
#include "prunex.h"

#include "files.h"
#include "logging.h"
#include "errorlogx.h"
#include "functions.h"
#include "osmparser.h"
#include "tagging.h"
#include "uncompress.h"


/* Global variables */

/*+ The name of the temporary directory. +*/
char *option_tmpdirname=NULL;

/*+ The amount of RAM to use for filesorting. +*/
size_t option_filesort_ramsize=0;

/*+ The number of threads to use for filesorting. +*/
int option_filesort_threads=1;


/* Local functions */

static void print_usage(int detail,const char *argerr,const char *err);


/*++++++++++++++++++++++++++++++++++++++
  The main program for the planetsplitter.
  ++++++++++++++++++++++++++++++++++++++*/

int main(int argc,char** argv)
{
 NodesX     *OSMNodes;
 SegmentsX  *OSMSegments,*SuperSegments=NULL,*MergedSegments=NULL;
 WaysX      *OSMWays;
 RelationsX *OSMRelations;
 int         iteration=0,quit=0;
 int         max_iterations=5;
 char       *dirname=NULL,*prefix=NULL,*tagging=NULL,*errorlog=NULL;
 int         option_parse_only=0,option_process_only=0;
 int         option_append=0,option_keep=0,option_changes=0;
 int         option_filenames=0;
 int         option_prune_isolated=500,option_prune_short=5,option_prune_straight=3;
 int         arg;

 printf_program_start();

 /* Parse the command line arguments */

 for(arg=1;arg<argc;arg++)
   {
    if(!strcmp(argv[arg],"--version"))
       print_usage(-1,NULL,NULL);
    else if(!strcmp(argv[arg],"--help"))
       print_usage(1,NULL,NULL);
    else if(!strncmp(argv[arg],"--dir=",6))
       dirname=&argv[arg][6];
    else if(!strncmp(argv[arg],"--prefix=",9))
       prefix=&argv[arg][9];
    else if(!strncmp(argv[arg],"--sort-ram-size=",16))
       option_filesort_ramsize=atoi(&argv[arg][16]);
#if defined(USE_PTHREADS) && USE_PTHREADS
    else if(!strncmp(argv[arg],"--sort-threads=",15))
       option_filesort_threads=atoi(&argv[arg][15]);
#endif
    else if(!strncmp(argv[arg],"--tmpdir=",9))
       option_tmpdirname=&argv[arg][9];
    else if(!strncmp(argv[arg],"--tagging=",10))
       tagging=&argv[arg][10];
    else if(!strcmp(argv[arg],"--loggable"))
       option_loggable=1;
    else if(!strcmp(argv[arg],"--logtime"))
       option_logtime=1;
    else if(!strcmp(argv[arg],"--logmemory"))
       option_logmemory=1;
    else if(!strcmp(argv[arg],"--errorlog"))
       errorlog="error.log";
    else if(!strncmp(argv[arg],"--errorlog=",11))
       errorlog=&argv[arg][11];
    else if(!strcmp(argv[arg],"--parse-only"))
       option_parse_only=1;
    else if(!strcmp(argv[arg],"--process-only"))
       option_process_only=1;
    else if(!strcmp(argv[arg],"--append"))
       option_append=1;
    else if(!strcmp(argv[arg],"--keep"))
       option_keep=1;
    else if(!strcmp(argv[arg],"--changes"))
       option_changes=1;
    else if(!strncmp(argv[arg],"--max-iterations=",17))
       max_iterations=atoi(&argv[arg][17]);
    else if(!strncmp(argv[arg],"--prune",7))
      {
       if(!strcmp(&argv[arg][7],"-none"))
          option_prune_isolated=option_prune_short=option_prune_straight=0;
       else if(!strncmp(&argv[arg][7],"-isolated=",10))
          option_prune_isolated=atoi(&argv[arg][17]);
       else if(!strncmp(&argv[arg][7],"-short=",7))
          option_prune_short=atoi(&argv[arg][14]);
       else if(!strncmp(&argv[arg][7],"-straight=",10))
          option_prune_straight=atoi(&argv[arg][17]);
       else
          print_usage(0,argv[arg],NULL);
      }
    else if(argv[arg][0]=='-' && argv[arg][1]=='-')
       print_usage(0,argv[arg],NULL);
    else
       option_filenames++;
   }

 /* Check the specified command line options */

 if(option_parse_only && option_process_only)
    print_usage(0,NULL,"Cannot use '--parse-only' and '--process-only' at the same time.");

 if(option_append && option_process_only)
    print_usage(0,NULL,"Cannot use '--append' and '--process-only' at the same time.");

 if(option_filenames && option_process_only)
    print_usage(0,NULL,"Cannot use '--process-only' and filenames at the same time.");

 if(!option_filenames && !option_process_only)
    print_usage(0,NULL,"File names must be specified unless using '--process-only'");

 if(!option_filesort_ramsize)
   {
#if SLIM
    option_filesort_ramsize=64*1024*1024;
#else
    option_filesort_ramsize=256*1024*1024;
#endif
   }
 else
    option_filesort_ramsize*=1024*1024;

 if(!option_tmpdirname)
   {
    if(!dirname)
       option_tmpdirname=".";
    else
       option_tmpdirname=dirname;
   }

 if(!option_process_only)
   {
    if(tagging)
      {
       if(!ExistsFile(tagging))
         {
          fprintf(stderr,"Error: The '--tagging' option specifies a file that does not exist.\n");
          exit(EXIT_FAILURE);
         }
      }
    else
      {
       tagging=FileName(dirname,prefix,"tagging.xml");

       if(!ExistsFile(tagging))
         {
          free(tagging);

          tagging=FileName(ROUTINO_DATADIR,NULL,"tagging.xml");

          if(!ExistsFile(tagging))
            {
             fprintf(stderr,"Error: The '--tagging' option was not used and the default 'tagging.xml' does not exist.\n");
             exit(EXIT_FAILURE);
            }
         }
      }

    if(ParseXMLTaggingRules(tagging))
      {
       fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging);
       exit(EXIT_FAILURE);
      }
   }

 /* Create new node, segment, way and relation variables */

 OSMNodes=NewNodeList(option_append||option_changes,option_process_only);

 OSMWays=NewWayList(option_append||option_changes,option_process_only);

 OSMRelations=NewRelationList(option_append||option_changes,option_process_only);

 /* Create the error log file */

 if(errorlog)
    open_errorlog(FileName(dirname,prefix,errorlog),option_append||option_changes||option_process_only,option_keep);

 /* Parse the file */

if(!option_process_only)
  {
   for(arg=1;arg<argc;arg++)
     {
      int fd;
      char *filename,*p;

      if(argv[arg][0]=='-' && argv[arg][1]=='-')
         continue;

      filename=strcpy(malloc(strlen(argv[arg])+1),argv[arg]);

      fd=OpenFile(filename);

      if((p=strstr(filename,".bz2")) && !strcmp(p,".bz2"))
        {
         fd=Uncompress_Bzip2(fd);
         *p=0;
        }

      if((p=strstr(filename,".gz")) && !strcmp(p,".gz"))
        {
         fd=Uncompress_Gzip(fd);
         *p=0;
        }

      if((p=strstr(filename,".xz")) && !strcmp(p,".xz"))
        {
         fd=Uncompress_Xz(fd);
         *p=0;
        }

      if(option_changes)
        {
         printf("\nParse OSC Data [%s]\n==============\n\n",filename);
         fflush(stdout);

         if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf"))
           {
            logassert(0,"Unable to read a PBF file to apply changes (format does not permit this)");
           }
         else if((p=strstr(filename,".o5c")) && !strcmp(p,".o5c"))
           {
            if(ParseO5CFile(fd,OSMNodes,OSMWays,OSMRelations))
               exit(EXIT_FAILURE);
           }
         else
           {
            if(ParseOSCFile(fd,OSMNodes,OSMWays,OSMRelations))
               exit(EXIT_FAILURE);
           }
        }
      else
        {
         printf("\nParse OSM Data [%s]\n==============\n\n",filename);
         fflush(stdout);

         if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf"))
           {
            if(ParsePBFFile(fd,OSMNodes,OSMWays,OSMRelations))
               exit(EXIT_FAILURE);
           }
         else if((p=strstr(filename,".o5m")) && !strcmp(p,".o5m"))
           {
            if(ParseO5MFile(fd,OSMNodes,OSMWays,OSMRelations))
               exit(EXIT_FAILURE);
           }
         else
           {
            if(ParseOSMFile(fd,OSMNodes,OSMWays,OSMRelations))
               exit(EXIT_FAILURE);
           }
        }

      CloseFile(fd);

      free(filename);
     }

   DeleteXMLTaggingRules();
  }

 FinishNodeList(OSMNodes);
 FinishWayList(OSMWays);
 FinishRelationList(OSMRelations);

 if(option_parse_only)
   {
    FreeNodeList(OSMNodes,1);
    FreeWayList(OSMWays,1);
    FreeRelationList(OSMRelations,1);

    exit(EXIT_SUCCESS);
   }


 /* Sort the data */

 printf("\nSort OSM Data\n=============\n\n");
 fflush(stdout);

 /* Sort the nodes, ways and relations */

 SortNodeList(OSMNodes);

 SortWayList(OSMWays);

 SortRelationList(OSMRelations);

 /* Process the data */

 printf("\nProcess OSM Data\n================\n\n");
 fflush(stdout);

 /* Remove non-highway nodes by looking through the ways (must be before splitting the ways) */

 RemoveNonHighwayNodes(OSMNodes,OSMWays,option_keep||option_changes);

 /* Separate the segments and way names and sort them (must be before processing the segments) */

 OSMSegments=SplitWays(OSMWays,OSMNodes,option_keep||option_changes);

 SortWayNames(OSMWays);

 SortSegmentList(OSMSegments);

 /* Process the segments and index them (must be before processing relations) */

 ProcessSegments(OSMSegments,OSMNodes,OSMWays);

 IndexSegments(OSMSegments,OSMNodes,OSMWays);

 /* Process the route relations and turn relations (must be before compacting the ways) */

 ProcessRouteRelations(OSMRelations,OSMWays,option_keep||option_changes);

 ProcessTurnRelations(OSMRelations,OSMNodes,OSMSegments,OSMWays,option_keep||option_changes);

 /* Compact the ways */

 CompactWayList(OSMWays,OSMSegments);

 /* Sort the nodes and segments geographically */

 SortNodeListGeographically(OSMNodes);

 SortSegmentListGeographically(OSMSegments,OSMNodes);

 /* Re-index the segments */

 IndexSegments(OSMSegments,OSMNodes,OSMWays);

 /* Sort the turn relations geographically */

 SortTurnRelationListGeographically(OSMRelations,OSMNodes,OSMSegments,0);

 /* Prune unwanted nodes/segments */

 if(option_prune_straight || option_prune_isolated || option_prune_short)
   {
    printf("\nPrune Unneeded Data\n===================\n\n");
    fflush(stdout);

    if(option_prune_straight)
      {
       StartPruning(OSMNodes,OSMSegments,OSMWays);

       PruneStraightHighwayNodes(OSMNodes,OSMSegments,OSMWays,option_prune_straight);

       FinishPruning(OSMNodes,OSMSegments,OSMWays);

       RemovePrunedNodes(OSMNodes,OSMSegments);
       RemovePrunedSegments(OSMSegments,OSMWays);
       CompactWayList(OSMWays,OSMSegments);
       RemovePrunedTurnRelations(OSMRelations,OSMNodes);

       IndexSegments(OSMSegments,OSMNodes,OSMWays);
      }

    if(option_prune_isolated)
      {
       StartPruning(OSMNodes,OSMSegments,OSMWays);

       PruneIsolatedRegions(OSMNodes,OSMSegments,OSMWays,option_prune_isolated);

       FinishPruning(OSMNodes,OSMSegments,OSMWays);

       RemovePrunedNodes(OSMNodes,OSMSegments);
       RemovePrunedSegments(OSMSegments,OSMWays);
       CompactWayList(OSMWays,OSMSegments);
       RemovePrunedTurnRelations(OSMRelations,OSMNodes);

       IndexSegments(OSMSegments,OSMNodes,OSMWays);
      }

    if(option_prune_short)
      {
       StartPruning(OSMNodes,OSMSegments,OSMWays);

       PruneShortSegments(OSMNodes,OSMSegments,OSMWays,option_prune_short);

       FinishPruning(OSMNodes,OSMSegments,OSMWays);

       RemovePrunedNodes(OSMNodes,OSMSegments);
       RemovePrunedSegments(OSMSegments,OSMWays);
       CompactWayList(OSMWays,OSMSegments);
       RemovePrunedTurnRelations(OSMRelations,OSMNodes);

       IndexSegments(OSMSegments,OSMNodes,OSMWays);
      }
   }

 /* Repeated iteration on Super-Nodes and Super-Segments */

 do
   {
    index_t nsuper;

    printf("\nProcess Super-Data (iteration %d)\n================================%s\n\n",iteration,iteration>9?"=":"");
    fflush(stdout);

    if(iteration==0)
      {
       /* Select the super-nodes */

       ChooseSuperNodes(OSMNodes,OSMSegments,OSMWays);

       /* Select the super-segments */

       SuperSegments=CreateSuperSegments(OSMNodes,OSMSegments,OSMWays);

       nsuper=OSMSegments->number;
      }
    else
      {
       SegmentsX *SuperSegments2;

       /* Index the super-segments */

       IndexSegments(SuperSegments,OSMNodes,OSMWays);

       /* Select the super-nodes */

       ChooseSuperNodes(OSMNodes,SuperSegments,OSMWays);

       /* Select the super-segments */

       SuperSegments2=CreateSuperSegments(OSMNodes,SuperSegments,OSMWays);

       nsuper=SuperSegments->number;

       FreeSegmentList(SuperSegments);

       SuperSegments=SuperSegments2;
      }

    /* Sort the super-segments and remove duplicates */

    DeduplicateSuperSegments(SuperSegments,OSMWays);

    /* Check for end condition */

    if(SuperSegments->number==nsuper)
       quit=1;

    iteration++;

    if(iteration>max_iterations)
       quit=1;
   }
 while(!quit);

 /* Combine the super-segments */

 printf("\nCombine Segments and Super-Segments\n===================================\n\n");
 fflush(stdout);

 /* Merge the super-segments */

 MergedSegments=MergeSuperSegments(OSMSegments,SuperSegments);

 FreeSegmentList(OSMSegments);

 FreeSegmentList(SuperSegments);

 OSMSegments=MergedSegments;

 /* Cross reference the nodes and segments */

 printf("\nCross-Reference Nodes and Segments\n==================================\n\n");
 fflush(stdout);

 /* Sort the nodes and segments geographically */

 SortNodeListGeographically(OSMNodes);

 SortSegmentListGeographically(OSMSegments,OSMNodes);

 /* Re-index the segments */

 IndexSegments(OSMSegments,OSMNodes,OSMWays);

 /* Sort the turn relations geographically */

 SortTurnRelationListGeographically(OSMRelations,OSMNodes,OSMSegments,1);

 /* Output the results */

 printf("\nWrite Out Database Files\n========================\n\n");
 fflush(stdout);

 /* Write out the nodes */

 SaveNodeList(OSMNodes,FileName(dirname,prefix,"nodes.mem"),OSMSegments);

 /* Write out the segments */

 SaveSegmentList(OSMSegments,FileName(dirname,prefix,"segments.mem"));

 /* Write out the ways */

 SaveWayList(OSMWays,FileName(dirname,prefix,"ways.mem"));

 /* Write out the relations */

 SaveRelationList(OSMRelations,FileName(dirname,prefix,"relations.mem"));

 /* Free the memory (delete the temporary files) */

 FreeSegmentList(OSMSegments);

 /* Close the error log file and process the data */

 if(errorlog)
   {
    close_errorlog();

    if(option_keep)
      {
       ErrorLogsX *OSMErrorLogs;

       printf("\nCreate Error Log\n================\n\n");
       fflush(stdout);

       OSMErrorLogs=NewErrorLogList();

       ProcessErrorLogs(OSMErrorLogs,OSMNodes,OSMWays,OSMRelations);

       SortErrorLogsGeographically(OSMErrorLogs);

       SaveErrorLogs(OSMErrorLogs,FileName(dirname,prefix,"errorlogs.mem"));

       FreeErrorLogList(OSMErrorLogs);
      }
   }

 /* Free the memory (delete the temporary files) */

 FreeNodeList(OSMNodes,0);
 FreeWayList(OSMWays,0);
 FreeRelationList(OSMRelations,0);

 printf("\n");
 fflush(stdout);

 printf_program_end();

 exit(EXIT_SUCCESS);
}


/*++++++++++++++++++++++++++++++++++++++
  Print out the usage information.

  int detail The level of detail to use: -1 = just version number, 0 = low detail, 1 = full details.

  const char *argerr The argument that gave the error (if there is one).

  const char *err Other error message (if there is one).
  ++++++++++++++++++++++++++++++++++++++*/

static void print_usage(int detail,const char *argerr,const char *err)
{
 if(detail<0)
   {
    fprintf(stderr,
            "Routino version " ROUTINO_VERSION " " ROUTINO_URL ".\n"
            );
   }

 if(detail>=0)
   {
    fprintf(stderr,
            "Usage: planetsplitter [--version]\n"
            "                      [--help]\n"
            "                      [--dir=<dirname>] [--prefix=<name>]\n"
#if defined(USE_PTHREADS) && USE_PTHREADS
            "                      [--sort-ram-size=<size>] [--sort-threads=<number>]\n"
#else
            "                      [--sort-ram-size=<size>]\n"
#endif
            "                      [--tmpdir=<dirname>]\n"
            "                      [--tagging=<filename>]\n"
            "                      [--loggable] [--logtime] [--logmemory]\n"
            "                      [--errorlog[=<name>]]\n"
            "                      [--parse-only | --process-only]\n"
            "                      [--append] [--keep] [--changes]\n"
            "                      [--max-iterations=<number>]\n"
            "                      [--prune-none]\n"
            "                      [--prune-isolated=<len>]\n"
            "                      [--prune-short=<len>]\n"
            "                      [--prune-straight=<len>]\n"
            "                      [<filename.osm> ... | <filename.osc> ...\n"
            "                       | <filename.pbf> ...\n"
            "                       | <filename.o5m> ... | <filename.o5c> ..."
#if defined(USE_BZIP2) && USE_BZIP2
            "\n                       | <filename.(osm|osc|o5m|o5c).bz2> ..."
#endif
#if defined(USE_GZIP) && USE_GZIP
            "\n                       | <filename.(osm|osc|o5m|o5c).gz> ..."
#endif
#if defined(USE_XZ) && USE_XZ
            "\n                       | <filename.(osm|osc|o5m|o5c).xz> ..."
#endif
            "]\n");

    if(argerr)
       fprintf(stderr,
               "\n"
               "Error with command line parameter: %s\n",argerr);

    if(err)
       fprintf(stderr,
               "\n"
               "Error: %s\n",err);
   }

 if(detail==1)
    fprintf(stderr,
            "\n"
            "--version                 Print the version of Routino.\n"
            "\n"
            "--help                    Prints this information.\n"
            "\n"
            "--dir=<dirname>           The directory containing the routing database.\n"
            "--prefix=<name>           The filename prefix for the routing database.\n"
            "\n"
            "--sort-ram-size=<size>    The amount of RAM (in MB) to use for data sorting\n"
#if SLIM
            "                          (defaults to 64MB otherwise.)\n"
#else
            "                          (defaults to 256MB otherwise.)\n"
#endif
#if defined(USE_PTHREADS) && USE_PTHREADS
            "--sort-threads=<number>   The number of threads to use for data sorting.\n"
#endif
            "\n"
            "--tmpdir=<dirname>        The directory name for temporary files.\n"
            "                          (defaults to the '--dir' option directory.)\n"
            "\n"
            "--tagging=<filename>      The name of the XML file containing the tagging rules\n"
            "                          (defaults to 'tagging.xml' with '--dir' and\n"
            "                           '--prefix' options or the file installed in\n"
            "                           '" ROUTINO_DATADIR "').\n"
            "\n"
            "--loggable                Print progress messages suitable for logging to file.\n"
            "--logtime                 Print the elapsed time for each processing step.\n"
            "--logmemory               Print the max allocated/mapped memory for each step.\n"
            "--errorlog[=<name>]       Log parsing errors to 'error.log' or the given name\n"
            "                          (the '--dir' and '--prefix' options are applied).\n"
            "\n"
            "--parse-only              Parse the OSM/OSC file(s) and store the results.\n"
            "--process-only            Process the stored results from previous option.\n"
            "--append                  Parse the OSM file(s) and append to existing results.\n"
            "--keep                    Keep the intermediate files after parsing & sorting.\n"
            "--changes                 Parse the data as an OSC file and apply the changes.\n"
            "\n"
            "--max-iterations=<number> The number of iterations for finding super-nodes\n"
            "                          (defaults to 5).\n"
            "\n"
            "--prune-none              Disable the prune options below, they are re-enabled\n"
            "                          by adding them to the command line after this option.\n"
            "--prune-isolated=<len>    Remove access from small disconnected segment groups\n"
            "                          (defaults to removing groups under 500m).\n"
            "--prune-short=<len>       Remove short segments (defaults to removing segments\n"
            "                          up to a maximum length of 5m).\n"
            "--prune-straight=<len>    Remove nodes in almost straight highways (defaults to\n"
            "                          removing nodes up to 3m offset from a straight line).\n"
            "\n"
            "<filename.osm>, <filename.osc>, <filename.pbf>, <filename.o5m>, <filename.o5c>\n"
            "                          The name(s) of the file(s) to read and parse.\n"
            "                          Filenames ending '.pbf' read as PBF, filenames ending\n"
            "                          '.o5m' or '.o5c' read as O5M/O5C, others as XML.\n"
#if defined(USE_BZIP2) && USE_BZIP2
            "                          Filenames ending '.bz2' will be bzip2 uncompressed.\n"
#endif
#if defined(USE_GZIP) && USE_GZIP
            "                          Filenames ending '.gz' will be gzip uncompressed.\n"
#endif
#if defined(USE_XZ) && USE_XZ
            "                          Filenames ending '.xz' will be xz uncompressed.\n"
#endif
            "\n"
            "<transport> defaults to all but can be set to:\n"
            "%s"
            "\n"
            "<highway> can be selected from:\n"
            "%s"
            "\n"
            "<property> can be selected from:\n"
            "%s",
            TransportList(),HighwayList(),PropertyList());

 exit(!detail);
}
1	/***************************************
2	OSM planet file splitter.
3
4	Part of the Routino routing software.
5	****************/ /****************
6	This file Copyright 2008-2015 Andrew M. Bishop
7
8	This program is free software: you can redistribute it and/or modify
9	it under the terms of the GNU Affero General Public License as published by
10	the Free Software Foundation, either version 3 of the License, or
11	(at your option) any later version.
12
13	This program is distributed in the hope that it will be useful,
14	but WITHOUT ANY WARRANTY; without even the implied warranty of
15	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16	GNU Affero General Public License for more details.
17
18	You should have received a copy of the GNU Affero General Public License
19	along with this program. If not, see <http://www.gnu.org/licenses/>.
20	***************************************/
21
22
23	#include <stdio.h>
24	#include <stdlib.h>
25	#include <string.h>
26	#include <errno.h>
27
28	#include "version.h"
29
30	#include "types.h"
31	#include "ways.h"
32
33	#include "typesx.h"
34	#include "nodesx.h"
35	#include "segmentsx.h"
36	#include "waysx.h"
37	#include "relationsx.h"
38	#include "superx.h"
39	#include "prunex.h"
40
41	#include "files.h"
42	#include "logging.h"
43	#include "errorlogx.h"
44	#include "functions.h"
45	#include "osmparser.h"
46	#include "tagging.h"
47	#include "uncompress.h"
48
49
50	/* Global variables */
51
52	/+ The name of the temporary directory. +/
53	char *option_tmpdirname=NULL;
54
55	/+ The amount of RAM to use for filesorting. +/
56	size_t option_filesort_ramsize=0;
57
58	/+ The number of threads to use for filesorting. +/
59	int option_filesort_threads=1;
60
61
62	/* Local functions */
63
64	static void print_usage(int detail,const char argerr,const char err);
65
66
67	/*++++++++++++++++++++++++++++++++++++++
68	The main program for the planetsplitter.
69	++++++++++++++++++++++++++++++++++++++*/
70
71	int main(int argc,char** argv)
72	{
73	NodesX *OSMNodes;
74	SegmentsX OSMSegments,SuperSegments=NULL,*MergedSegments=NULL;
75	WaysX *OSMWays;
76	RelationsX *OSMRelations;
77	int iteration=0,quit=0;
78	int max_iterations=5;
79	char dirname=NULL,prefix=NULL,tagging=NULL,errorlog=NULL;
80	int option_parse_only=0,option_process_only=0;
81	int option_append=0,option_keep=0,option_changes=0;
82	int option_filenames=0;
83	int option_prune_isolated=500,option_prune_short=5,option_prune_straight=3;
84	int arg;
85
86	printf_program_start();
87
88	/* Parse the command line arguments */
89
90	for(arg=1;arg<argc;arg++)
91	{
92	if(!strcmp(argv[arg],"--version"))
93	print_usage(-1,NULL,NULL);
94	else if(!strcmp(argv[arg],"--help"))
95	print_usage(1,NULL,NULL);
96	else if(!strncmp(argv[arg],"--dir=",6))
97	dirname=&argv[arg][6];
98	else if(!strncmp(argv[arg],"--prefix=",9))
99	prefix=&argv[arg][9];
100	else if(!strncmp(argv[arg],"--sort-ram-size=",16))
101	option_filesort_ramsize=atoi(&argv[arg][16]);
102	#if defined(USE_PTHREADS) && USE_PTHREADS
103	else if(!strncmp(argv[arg],"--sort-threads=",15))
104	option_filesort_threads=atoi(&argv[arg][15]);
105	#endif
106	else if(!strncmp(argv[arg],"--tmpdir=",9))
107	option_tmpdirname=&argv[arg][9];
108	else if(!strncmp(argv[arg],"--tagging=",10))
109	tagging=&argv[arg][10];
110	else if(!strcmp(argv[arg],"--loggable"))
111	option_loggable=1;
112	else if(!strcmp(argv[arg],"--logtime"))
113	option_logtime=1;
114	else if(!strcmp(argv[arg],"--logmemory"))
115	option_logmemory=1;
116	else if(!strcmp(argv[arg],"--errorlog"))
117	errorlog="error.log";
118	else if(!strncmp(argv[arg],"--errorlog=",11))
119	errorlog=&argv[arg][11];
120	else if(!strcmp(argv[arg],"--parse-only"))
121	option_parse_only=1;
122	else if(!strcmp(argv[arg],"--process-only"))
123	option_process_only=1;
124	else if(!strcmp(argv[arg],"--append"))
125	option_append=1;
126	else if(!strcmp(argv[arg],"--keep"))
127	option_keep=1;
128	else if(!strcmp(argv[arg],"--changes"))
129	option_changes=1;
130	else if(!strncmp(argv[arg],"--max-iterations=",17))
131	max_iterations=atoi(&argv[arg][17]);
132	else if(!strncmp(argv[arg],"--prune",7))
133	{
134	if(!strcmp(&argv[arg][7],"-none"))
135	option_prune_isolated=option_prune_short=option_prune_straight=0;
136	else if(!strncmp(&argv[arg][7],"-isolated=",10))
137	option_prune_isolated=atoi(&argv[arg][17]);
138	else if(!strncmp(&argv[arg][7],"-short=",7))
139	option_prune_short=atoi(&argv[arg][14]);
140	else if(!strncmp(&argv[arg][7],"-straight=",10))
141	option_prune_straight=atoi(&argv[arg][17]);
142	else
143	print_usage(0,argv[arg],NULL);
144	}
145	else if(argv[arg][0]=='-' && argv[arg][1]=='-')
146	print_usage(0,argv[arg],NULL);
147	else
148	option_filenames++;
149	}
150
151	/* Check the specified command line options */
152
153	if(option_parse_only && option_process_only)
154	print_usage(0,NULL,"Cannot use '--parse-only' and '--process-only' at the same time.");
155
156	if(option_append && option_process_only)
157	print_usage(0,NULL,"Cannot use '--append' and '--process-only' at the same time.");
158
159	if(option_filenames && option_process_only)
160	print_usage(0,NULL,"Cannot use '--process-only' and filenames at the same time.");
161
162	if(!option_filenames && !option_process_only)
163	print_usage(0,NULL,"File names must be specified unless using '--process-only'");
164
165	if(!option_filesort_ramsize)
166	{
167	#if SLIM
168	option_filesort_ramsize=6410241024;
169	#else
170	option_filesort_ramsize=25610241024;
171	#endif
172	}
173	else
174	option_filesort_ramsize=10241024;
175
176	if(!option_tmpdirname)
177	{
178	if(!dirname)
179	option_tmpdirname=".";
180	else
181	option_tmpdirname=dirname;
182	}
183
184	if(!option_process_only)
185	{
186	if(tagging)
187	{
188	if(!ExistsFile(tagging))
189	{
190	fprintf(stderr,"Error: The '--tagging' option specifies a file that does not exist.\n");
191	exit(EXIT_FAILURE);
192	}
193	}
194	else
195	{
196	tagging=FileName(dirname,prefix,"tagging.xml");
197
198	if(!ExistsFile(tagging))
199	{
200	free(tagging);
201
202	tagging=FileName(ROUTINO_DATADIR,NULL,"tagging.xml");
203
204	if(!ExistsFile(tagging))
205	{
206	fprintf(stderr,"Error: The '--tagging' option was not used and the default 'tagging.xml' does not exist.\n");
207	exit(EXIT_FAILURE);
208	}
209	}
210	}
211
212	if(ParseXMLTaggingRules(tagging))
213	{
214	fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging);
215	exit(EXIT_FAILURE);
216	}
217	}
218
219	/* Create new node, segment, way and relation variables */
220
221	OSMNodes=NewNodeList(option_append\|\|option_changes,option_process_only);
222
223	OSMWays=NewWayList(option_append\|\|option_changes,option_process_only);
224
225	OSMRelations=NewRelationList(option_append\|\|option_changes,option_process_only);
226
227	/* Create the error log file */
228
229	if(errorlog)
230	open_errorlog(FileName(dirname,prefix,errorlog),option_append\|\|option_changes\|\|option_process_only,option_keep);
231
232	/* Parse the file */
233
234	if(!option_process_only)
235	{
236	for(arg=1;arg<argc;arg++)
237	{
238	int fd;
239	char filename,p;
240
241	if(argv[arg][0]=='-' && argv[arg][1]=='-')
242	continue;
243
244	filename=strcpy(malloc(strlen(argv[arg])+1),argv[arg]);
245
246	fd=OpenFile(filename);
247
248	if((p=strstr(filename,".bz2")) && !strcmp(p,".bz2"))
249	{
250	fd=Uncompress_Bzip2(fd);
251	*p=0;
252	}
253
254	if((p=strstr(filename,".gz")) && !strcmp(p,".gz"))
255	{
256	fd=Uncompress_Gzip(fd);
257	*p=0;
258	}
259
260	if((p=strstr(filename,".xz")) && !strcmp(p,".xz"))
261	{
262	fd=Uncompress_Xz(fd);
263	*p=0;
264	}
265
266	if(option_changes)
267	{
268	printf("\nParse OSC Data [%s]\n==============\n\n",filename);
269	fflush(stdout);
270
271	if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf"))
272	{
273	logassert(0,"Unable to read a PBF file to apply changes (format does not permit this)");
274	}
275	else if((p=strstr(filename,".o5c")) && !strcmp(p,".o5c"))
276	{
277	if(ParseO5CFile(fd,OSMNodes,OSMWays,OSMRelations))
278	exit(EXIT_FAILURE);
279	}
280	else
281	{
282	if(ParseOSCFile(fd,OSMNodes,OSMWays,OSMRelations))
283	exit(EXIT_FAILURE);
284	}
285	}
286	else
287	{
288	printf("\nParse OSM Data [%s]\n==============\n\n",filename);
289	fflush(stdout);
290
291	if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf"))
292	{
293	if(ParsePBFFile(fd,OSMNodes,OSMWays,OSMRelations))
294	exit(EXIT_FAILURE);
295	}
296	else if((p=strstr(filename,".o5m")) && !strcmp(p,".o5m"))
297	{
298	if(ParseO5MFile(fd,OSMNodes,OSMWays,OSMRelations))
299	exit(EXIT_FAILURE);
300	}
301	else
302	{
303	if(ParseOSMFile(fd,OSMNodes,OSMWays,OSMRelations))
304	exit(EXIT_FAILURE);
305	}
306	}
307
308	CloseFile(fd);
309
310	free(filename);
311	}
312
313	DeleteXMLTaggingRules();
314	}
315
316	FinishNodeList(OSMNodes);
317	FinishWayList(OSMWays);
318	FinishRelationList(OSMRelations);
319
320	if(option_parse_only)
321	{
322	FreeNodeList(OSMNodes,1);
323	FreeWayList(OSMWays,1);
324	FreeRelationList(OSMRelations,1);
325
326	exit(EXIT_SUCCESS);
327	}
328
329
330	/* Sort the data */
331
332	printf("\nSort OSM Data\n=============\n\n");
333	fflush(stdout);
334
335	/* Sort the nodes, ways and relations */
336
337	SortNodeList(OSMNodes);
338
339	SortWayList(OSMWays);
340
341	SortRelationList(OSMRelations);
342
343	/* Process the data */
344
345	printf("\nProcess OSM Data\n================\n\n");
346	fflush(stdout);
347
348	/* Remove non-highway nodes by looking through the ways (must be before splitting the ways) */
349
350	RemoveNonHighwayNodes(OSMNodes,OSMWays,option_keep\|\|option_changes);
351
352	/* Separate the segments and way names and sort them (must be before processing the segments) */
353
354	OSMSegments=SplitWays(OSMWays,OSMNodes,option_keep\|\|option_changes);
355
356	SortWayNames(OSMWays);
357
358	SortSegmentList(OSMSegments);
359
360	/* Process the segments and index them (must be before processing relations) */
361
362	ProcessSegments(OSMSegments,OSMNodes,OSMWays);
363
364	IndexSegments(OSMSegments,OSMNodes,OSMWays);
365
366	/* Process the route relations and turn relations (must be before compacting the ways) */
367
368	ProcessRouteRelations(OSMRelations,OSMWays,option_keep\|\|option_changes);
369
370	ProcessTurnRelations(OSMRelations,OSMNodes,OSMSegments,OSMWays,option_keep\|\|option_changes);
371
372	/* Compact the ways */
373
374	CompactWayList(OSMWays,OSMSegments);
375
376	/* Sort the nodes and segments geographically */
377
378	SortNodeListGeographically(OSMNodes);
379
380	SortSegmentListGeographically(OSMSegments,OSMNodes);
381
382	/* Re-index the segments */
383
384	IndexSegments(OSMSegments,OSMNodes,OSMWays);
385
386	/* Sort the turn relations geographically */
387
388	SortTurnRelationListGeographically(OSMRelations,OSMNodes,OSMSegments,0);
389
390	/* Prune unwanted nodes/segments */
391
392	if(option_prune_straight \|\| option_prune_isolated \|\| option_prune_short)
393	{
394	printf("\nPrune Unneeded Data\n===================\n\n");
395	fflush(stdout);
396
397	if(option_prune_straight)
398	{
399	StartPruning(OSMNodes,OSMSegments,OSMWays);
400
401	PruneStraightHighwayNodes(OSMNodes,OSMSegments,OSMWays,option_prune_straight);
402
403	FinishPruning(OSMNodes,OSMSegments,OSMWays);
404
405	RemovePrunedNodes(OSMNodes,OSMSegments);
406	RemovePrunedSegments(OSMSegments,OSMWays);
407	CompactWayList(OSMWays,OSMSegments);
408	RemovePrunedTurnRelations(OSMRelations,OSMNodes);
409
410	IndexSegments(OSMSegments,OSMNodes,OSMWays);
411	}
412
413	if(option_prune_isolated)
414	{
415	StartPruning(OSMNodes,OSMSegments,OSMWays);
416
417	PruneIsolatedRegions(OSMNodes,OSMSegments,OSMWays,option_prune_isolated);
418
419	FinishPruning(OSMNodes,OSMSegments,OSMWays);
420
421	RemovePrunedNodes(OSMNodes,OSMSegments);
422	RemovePrunedSegments(OSMSegments,OSMWays);
423	CompactWayList(OSMWays,OSMSegments);
424	RemovePrunedTurnRelations(OSMRelations,OSMNodes);
425
426	IndexSegments(OSMSegments,OSMNodes,OSMWays);
427	}
428
429	if(option_prune_short)
430	{
431	StartPruning(OSMNodes,OSMSegments,OSMWays);
432
433	PruneShortSegments(OSMNodes,OSMSegments,OSMWays,option_prune_short);
434
435	FinishPruning(OSMNodes,OSMSegments,OSMWays);
436
437	RemovePrunedNodes(OSMNodes,OSMSegments);
438	RemovePrunedSegments(OSMSegments,OSMWays);
439	CompactWayList(OSMWays,OSMSegments);
440	RemovePrunedTurnRelations(OSMRelations,OSMNodes);
441
442	IndexSegments(OSMSegments,OSMNodes,OSMWays);
443	}
444	}
445
446	/* Repeated iteration on Super-Nodes and Super-Segments */
447
448	do
449	{
450	index_t nsuper;
451
452	printf("\nProcess Super-Data (iteration %d)\n================================%s\n\n",iteration,iteration>9?"=":"");
453	fflush(stdout);
454
455	if(iteration==0)
456	{
457	/* Select the super-nodes */
458
459	ChooseSuperNodes(OSMNodes,OSMSegments,OSMWays);
460
461	/* Select the super-segments */
462
463	SuperSegments=CreateSuperSegments(OSMNodes,OSMSegments,OSMWays);
464
465	nsuper=OSMSegments->number;
466	}
467	else
468	{
469	SegmentsX *SuperSegments2;
470
471	/* Index the super-segments */
472
473	IndexSegments(SuperSegments,OSMNodes,OSMWays);
474
475	/* Select the super-nodes */
476
477	ChooseSuperNodes(OSMNodes,SuperSegments,OSMWays);
478
479	/* Select the super-segments */
480
481	SuperSegments2=CreateSuperSegments(OSMNodes,SuperSegments,OSMWays);
482
483	nsuper=SuperSegments->number;
484
485	FreeSegmentList(SuperSegments);
486
487	SuperSegments=SuperSegments2;
488	}
489
490	/* Sort the super-segments and remove duplicates */
491
492	DeduplicateSuperSegments(SuperSegments,OSMWays);
493
494	/* Check for end condition */
495
496	if(SuperSegments->number==nsuper)
497	quit=1;
498
499	iteration++;
500
501	if(iteration>max_iterations)
502	quit=1;
503	}
504	while(!quit);
505
506	/* Combine the super-segments */
507
508	printf("\nCombine Segments and Super-Segments\n===================================\n\n");
509	fflush(stdout);
510
511	/* Merge the super-segments */
512
513	MergedSegments=MergeSuperSegments(OSMSegments,SuperSegments);
514
515	FreeSegmentList(OSMSegments);
516
517	FreeSegmentList(SuperSegments);
518
519	OSMSegments=MergedSegments;
520
521	/* Cross reference the nodes and segments */
522
523	printf("\nCross-Reference Nodes and Segments\n==================================\n\n");
524	fflush(stdout);
525
526	/* Sort the nodes and segments geographically */
527
528	SortNodeListGeographically(OSMNodes);
529
530	SortSegmentListGeographically(OSMSegments,OSMNodes);
531
532	/* Re-index the segments */
533
534	IndexSegments(OSMSegments,OSMNodes,OSMWays);
535
536	/* Sort the turn relations geographically */
537
538	SortTurnRelationListGeographically(OSMRelations,OSMNodes,OSMSegments,1);
539
540	/* Output the results */
541
542	printf("\nWrite Out Database Files\n========================\n\n");
543	fflush(stdout);
544
545	/* Write out the nodes */
546
547	SaveNodeList(OSMNodes,FileName(dirname,prefix,"nodes.mem"),OSMSegments);
548
549	/* Write out the segments */
550
551	SaveSegmentList(OSMSegments,FileName(dirname,prefix,"segments.mem"));
552
553	/* Write out the ways */
554
555	SaveWayList(OSMWays,FileName(dirname,prefix,"ways.mem"));
556
557	/* Write out the relations */
558
559	SaveRelationList(OSMRelations,FileName(dirname,prefix,"relations.mem"));
560
561	/* Free the memory (delete the temporary files) */
562
563	FreeSegmentList(OSMSegments);
564
565	/* Close the error log file and process the data */
566
567	if(errorlog)
568	{
569	close_errorlog();
570
571	if(option_keep)
572	{
573	ErrorLogsX *OSMErrorLogs;
574
575	printf("\nCreate Error Log\n================\n\n");
576	fflush(stdout);
577
578	OSMErrorLogs=NewErrorLogList();
579
580	ProcessErrorLogs(OSMErrorLogs,OSMNodes,OSMWays,OSMRelations);
581
582	SortErrorLogsGeographically(OSMErrorLogs);
583
584	SaveErrorLogs(OSMErrorLogs,FileName(dirname,prefix,"errorlogs.mem"));
585
586	FreeErrorLogList(OSMErrorLogs);
587	}
588	}
589
590	/* Free the memory (delete the temporary files) */
591
592	FreeNodeList(OSMNodes,0);
593	FreeWayList(OSMWays,0);
594	FreeRelationList(OSMRelations,0);
595
596	printf("\n");
597	fflush(stdout);
598
599	printf_program_end();
600
601	exit(EXIT_SUCCESS);
602	}
603
604
605	/*++++++++++++++++++++++++++++++++++++++
606	Print out the usage information.
607
608	int detail The level of detail to use: -1 = just version number, 0 = low detail, 1 = full details.
609
610	const char *argerr The argument that gave the error (if there is one).
611
612	const char *err Other error message (if there is one).
613	++++++++++++++++++++++++++++++++++++++*/
614
615	static void print_usage(int detail,const char argerr,const char err)
616	{
617	if(detail<0)
618	{
619	fprintf(stderr,
620	"Routino version " ROUTINO_VERSION " " ROUTINO_URL ".\n"
621	);
622	}
623
624	if(detail>=0)
625	{
626	fprintf(stderr,
627	"Usage: planetsplitter [--version]\n"
628	" [--help]\n"
629	" [--dir=<dirname>] [--prefix=<name>]\n"
630	#if defined(USE_PTHREADS) && USE_PTHREADS
631	" [--sort-ram-size=<size>] [--sort-threads=<number>]\n"
632	#else
633	" [--sort-ram-size=<size>]\n"
634	#endif
635	" [--tmpdir=<dirname>]\n"
636	" [--tagging=<filename>]\n"
637	" [--loggable] [--logtime] [--logmemory]\n"
638	" [--errorlog[=<name>]]\n"
639	" [--parse-only \| --process-only]\n"
640	" [--append] [--keep] [--changes]\n"
641	" [--max-iterations=<number>]\n"
642	" [--prune-none]\n"
643	" [--prune-isolated=<len>]\n"
644	" [--prune-short=<len>]\n"
645	" [--prune-straight=<len>]\n"
646	" [<filename.osm> ... \| <filename.osc> ...\n"
647	" \| <filename.pbf> ...\n"
648	" \| <filename.o5m> ... \| <filename.o5c> ..."
649	#if defined(USE_BZIP2) && USE_BZIP2
650	"\n \| <filename.(osm\|osc\|o5m\|o5c).bz2> ..."
651	#endif
652	#if defined(USE_GZIP) && USE_GZIP
653	"\n \| <filename.(osm\|osc\|o5m\|o5c).gz> ..."
654	#endif
655	#if defined(USE_XZ) && USE_XZ
656	"\n \| <filename.(osm\|osc\|o5m\|o5c).xz> ..."
657	#endif
658	"]\n");
659
660	if(argerr)
661	fprintf(stderr,
662	"\n"
663	"Error with command line parameter: %s\n",argerr);
664
665	if(err)
666	fprintf(stderr,
667	"\n"
668	"Error: %s\n",err);
669	}
670
671	if(detail==1)
672	fprintf(stderr,
673	"\n"
674	"--version Print the version of Routino.\n"
675	"\n"
676	"--help Prints this information.\n"
677	"\n"
678	"--dir=<dirname> The directory containing the routing database.\n"
679	"--prefix=<name> The filename prefix for the routing database.\n"
680	"\n"
681	"--sort-ram-size=<size> The amount of RAM (in MB) to use for data sorting\n"
682	#if SLIM
683	" (defaults to 64MB otherwise.)\n"
684	#else
685	" (defaults to 256MB otherwise.)\n"
686	#endif
687	#if defined(USE_PTHREADS) && USE_PTHREADS
688	"--sort-threads=<number> The number of threads to use for data sorting.\n"
689	#endif
690	"\n"
691	"--tmpdir=<dirname> The directory name for temporary files.\n"
692	" (defaults to the '--dir' option directory.)\n"
693	"\n"
694	"--tagging=<filename> The name of the XML file containing the tagging rules\n"
695	" (defaults to 'tagging.xml' with '--dir' and\n"
696	" '--prefix' options or the file installed in\n"
697	" '" ROUTINO_DATADIR "').\n"
698	"\n"
699	"--loggable Print progress messages suitable for logging to file.\n"
700	"--logtime Print the elapsed time for each processing step.\n"
701	"--logmemory Print the max allocated/mapped memory for each step.\n"
702	"--errorlog[=<name>] Log parsing errors to 'error.log' or the given name\n"
703	" (the '--dir' and '--prefix' options are applied).\n"
704	"\n"
705	"--parse-only Parse the OSM/OSC file(s) and store the results.\n"
706	"--process-only Process the stored results from previous option.\n"
707	"--append Parse the OSM file(s) and append to existing results.\n"
708	"--keep Keep the intermediate files after parsing & sorting.\n"
709	"--changes Parse the data as an OSC file and apply the changes.\n"
710	"\n"
711	"--max-iterations=<number> The number of iterations for finding super-nodes\n"
712	" (defaults to 5).\n"
713	"\n"
714	"--prune-none Disable the prune options below, they are re-enabled\n"
715	" by adding them to the command line after this option.\n"
716	"--prune-isolated=<len> Remove access from small disconnected segment groups\n"
717	" (defaults to removing groups under 500m).\n"
718	"--prune-short=<len> Remove short segments (defaults to removing segments\n"
719	" up to a maximum length of 5m).\n"
720	"--prune-straight=<len> Remove nodes in almost straight highways (defaults to\n"
721	" removing nodes up to 3m offset from a straight line).\n"
722	"\n"
723	"<filename.osm>, <filename.osc>, <filename.pbf>, <filename.o5m>, <filename.o5c>\n"
724	" The name(s) of the file(s) to read and parse.\n"
725	" Filenames ending '.pbf' read as PBF, filenames ending\n"
726	" '.o5m' or '.o5c' read as O5M/O5C, others as XML.\n"
727	#if defined(USE_BZIP2) && USE_BZIP2
728	" Filenames ending '.bz2' will be bzip2 uncompressed.\n"
729	#endif
730	#if defined(USE_GZIP) && USE_GZIP
731	" Filenames ending '.gz' will be gzip uncompressed.\n"
732	#endif
733	#if defined(USE_XZ) && USE_XZ
734	" Filenames ending '.xz' will be xz uncompressed.\n"
735	#endif
736	"\n"
737	"<transport> defaults to all but can be set to:\n"
738	"%s"
739	"\n"
740	"<highway> can be selected from:\n"
741	"%s"
742	"\n"
743	"<property> can be selected from:\n"
744	"%s",
745	TransportList(),HighwayList(),PropertyList());
746
747	exit(!detail);
748	}