Package docker image within mapdata by smaller layers (#99)

* chore: also publish wayid2nodeids.csv.snappy seperately

* chore: osrm-backend-within-mapdata docker

* chore: export compiled files directly instead of package

* chore: pass data version when compile

* fix: typo

* chore: fix permission issue

* docs: udpate and cleanup

* docs: fix typo

docker-orchestration/README.md

@@ -3,12 +3,23 @@
 ## Docker Images 
 ### osrm-backend-dev
 Base image for telenav osrm-backend development, include all building and running dependencies.     
-See details in [osrm-backend-dev docker](./osrm-backend-dev/).    
+```bash
+$ docker pull telenavmap/osrm-backend-dev 
+```
 
 ### osrm-backend
 Image within built osrm binaries(`osrm-extract/osrm-partition/osrm-customize/...`) and running dependencies. It can be used to **compile data** or **startup routed**.      
+
+```bash
+$ docker pull telenavmap/osrm-backend 
+```
+
 See details in [osrm-backend docker](./osrm-backend/)
 
+### osrm-backend-within-mapdata
+Image based on [osrm-backend docker image](#osrm-backend) and put compiled mapdata inside.          
+NOTE: It's a temporary workaround for easily run in k8s. It's NOT a good idea to put mapdata in image directly since the map data is too big. For long-term, discussing in [#93](https://github.com/Telenav/osrm-backend/issues/93).      
+
 ### osrm-frontend
 Image contains web tool to check routing and guidance result.  
 It uses MapBox GL JS and apply routing response on top of Mapbox vector tiles.  

docker-orchestration/osrm-backend-within-mapdata/Dockerfile

@@ -0,0 +1,37 @@
+
+
+ARG FROM_TAG=latest
+FROM telenavmap/osrm-backend:${FROM_TAG} 
+
+
+COPY ./map/map.osrm.cell_metrics /osrm-data/
+COPY ./map/map.osrm.cells /osrm-data/
+COPY ./map/map.osrm.cnbg /osrm-data/
+COPY ./map/map.osrm.cnbg_to_ebg /osrm-data/
+COPY ./map/map.osrm.datasource_names /osrm-data/
+COPY ./map/map.osrm.ebg /osrm-data/
+COPY ./map/map.osrm.ebg_nodes /osrm-data/
+COPY ./map/map.osrm.edges /osrm-data/
+COPY ./map/map.osrm.enw /osrm-data/
+COPY ./map/map.osrm.fileIndex /osrm-data/
+COPY ./map/map.osrm.geometry /osrm-data/
+COPY ./map/map.osrm.icd /osrm-data/
+COPY ./map/map.osrm.maneuver_overrides /osrm-data/
+COPY ./map/map.osrm.mldgr /osrm-data/
+COPY ./map/map.osrm.names /osrm-data/
+COPY ./map/map.osrm.nbg_nodes /osrm-data/
+COPY ./map/map.osrm.partition /osrm-data/
+COPY ./map/map.osrm.properties /osrm-data/
+COPY ./map/map.osrm.ramIndex /osrm-data/
+COPY ./map/map.osrm.restrictions /osrm-data/
+COPY ./map/map.osrm.timestamp /osrm-data/
+COPY ./map/map.osrm.tld /osrm-data/
+COPY ./map/map.osrm.tls /osrm-data/
+COPY ./map/map.osrm.turn_duration_penalties /osrm-data/
+COPY ./map/map.osrm.turn_penalties_index /osrm-data/
+COPY ./map/map.osrm.turn_weight_penalties /osrm-data/
+COPY ./map/wayid2nodeids.csv.snappy /osrm-data/
+
+EXPOSE 5000
+ENTRYPOINT ["/docker-entrypoint.sh"]
+

docker-orchestration/osrm-backend/README.md

@@ -1,34 +1,13 @@
 # telenav osrm-backend docker
 Image within built osrm binaries(`osrm-extract/osrm-partition/osrm-customize/...`) and running dependencies. It can be used to **compile data** or **startup routed**.      
 
-## Build Image
-Please use below jenkins job to build docker image within osrm binaries from source code.    
-
-- [(Telenav Internal) Jenkins Job - Build_Telenav_OSRM_Backend_Docker](https://shd-routingfp-01.telenav.cn:8443/view/OSRM/job/Build_Telenav_OSRM_Backend_Docker/)    
-
-## Compile Data
-Please use below jenkins job to compile mapdata from PBF to osrm data.       
-There're two options for publish the compiled osrm data:    
-- generate new docker image from current one within compile data
-- export as `map.tar.gz`    
-
-See below job for details:      
-- [(Telenav Internal) Jenkins Job - Compile_Mapdata_In_Telenav_OSRM_Backend_Docker](https://shd-routingfp-01.telenav.cn:8443/view/OSRM/job/Compile_Mapdata_In_Telenav_OSRM_Backend_Docker/)    
-
-
-## Run 
-### Run with osrm-data inside
+## Image
 
 ```bash
-$ docker pull artifactory.telenav.com/telenav-docker-preprod/osrm-backend:8__9__master-telenav__20190618T135121CST__20190514T140904CST-california-latest
-$ 
-# run with default trafficproxy
-$ docker run -d -p 5000:5000 artifactory.telenav.com/telenav-docker-preprod/osrm-backend:8__9__master-telenav__20190618T135121CST__20190514T140904CST-california-latest routed_startup 
-
-# run with another trafficproxy(e.g. 10.189.102.81)
-$ docker run -d -p 5000:5000 artifactory.telenav.com/telenav-docker-preprod/osrm-backend:8__9__master-telenav__20190618T135121CST__20190514T140904CST-california-latest routed_startup 10.189.102.81
+$ docker pull telenavmap/osrm-backend 
 ```
 
+## Run 
 ### Run with osrm-data outside
 
 ```bash
@@ -66,7 +45,7 @@ $ cd ..
 
 # pull & run
 $ docker pull telenavmap/osrm-backend:no.63-20191112-master-telenav-845f953
-$ docker run -d -p 5000:5000  "src=$(pwd)/osrm-data,dst=/osrm-data,type=bind" telenavmap/osrm-backend:no.63-20191112-master-telenav-845f953 routed_startup 
+$ docker run -d -p 5000:5000  "src=$(pwd)/osrm-data,dst=/osrm-data,type=bind" telenavmap/osrm-backend:no.63-20191112-master-telenav-845f953 routed_no_traffic_startup 
 ```
 
 ## Example By Manual

docker-orchestration/osrm-backend/docker-entrypoint.sh

@@ -46,16 +46,8 @@ elif [ "$1" = 'compile_mapdata' ]; then
   #trap _sig SIGKILL SIGTERM SIGHUP SIGINT EXIT
 
   PBF_FILE_URL=${2}
-  KEEP_COMPILED_DATA=${3:-"false"}
-  GENERATE_DATA_PACKAGE=${4:-"false"}
-  IS_TELENAV_PBF=${5:-"false"}
-
-  # use PBF file name + IMAGE_TAG as data_version which can be returned in each JSON response
-  DATA_VERSION=`echo ${PBF_FILE_URL} | rev | cut -d / -f 1 | rev`
-  if [ x${IMAGE_TAG} != x ]; then
-    DATA_VERSION=${DATA_VERSION}--compiled-by-${IMAGE_TAG}
-  fi
-  echo ${DATA_VERSION} 
+  IS_TELENAV_PBF=${3:-"false"}
+  DATA_VERSION=${4:-"unset"}
 
   curl -sSL -f ${PBF_FILE_URL} > $DATA_PATH/${MAPDATA_NAME_WITH_SUFFIX}.osm.pbf
   ${BUILD_PATH}/osrm-extract $DATA_PATH/${MAPDATA_NAME_WITH_SUFFIX}.osm.pbf -p ${BUILD_PATH}/profiles/car.lua -d ${DATA_VERSION} ${OSRM_EXTRA_COMMAND}
@@ -65,25 +57,15 @@ elif [ "$1" = 'compile_mapdata' ]; then
   ${BUILD_PATH}/snappy -i $DATA_PATH/${WAYID2NODEIDS_MAPPING_FILE} -o $DATA_PATH/${WAYID2NODEIDS_MAPPING_FILE_COMPRESSED}
   ls -lh $DATA_PATH/
 
-  # clean source pbf and temp .osrm
+  # clean source pbf and temp files
   rm -f $DATA_PATH/${MAPDATA_NAME_WITH_SUFFIX}.osm.pbf
   rm -f $DATA_PATH/${MAPDATA_NAME_WITH_SUFFIX}.osrm
   rm -f $DATA_PATH/${WAYID2NODEIDS_MAPPING_FILE}
 
-  # package and publish compiled mapdata 
-  if [ ${GENERATE_DATA_PACKAGE} == "true" ]; then
-    cd ${DATA_PATH}
-    tar -zcf ${MAPDATA_NAME_WITH_SUFFIX}.tar.gz *
-
-    SAVE_DATA_PACKAGE_PATH=/save-data
-    mkdir -p ${SAVE_DATA_PACKAGE_PATH}
-    mv ${DATA_PATH}/${MAPDATA_NAME_WITH_SUFFIX}.tar.gz ${SAVE_DATA_PACKAGE_PATH}/
-  fi
-
-  # rm compiled data if not needed
-  if [ ${KEEP_COMPILED_DATA} != "true" ]; then
-    rm -f $DATA_PATH/*
-  fi
+  # export compiled mapdata to mounted path for publishing 
+  SAVE_DATA_PACKAGE_PATH=/save-data
+  mv ${DATA_PATH}/* ${SAVE_DATA_PACKAGE_PATH}/
+  chmod 777 ${SAVE_DATA_PACKAGE_PATH}/*
 
 else
   exec "$@"