From 1536f6fe31420021b6ac8ed3133acdc3f7569d72 Mon Sep 17 00:00:00 2001
From: David Hoese <david.hoese@ssec.wisc.edu>
Date: Thu, 16 Jan 2020 15:08:03 -0600
Subject: [PATCH] Fix tile generation not re-using existing geotiffs
---
tile_gen/Dockerfile | 2 ++
tile_gen/generate_tiles.py | 12 ++++++++++--
tile_gen/run.sh | 2 ++
3 files changed, 14 insertions(+), 2 deletions(-)
diff --git a/tile_gen/Dockerfile b/tile_gen/Dockerfile
index 7d7d9e8..125540c 100644
--- a/tile_gen/Dockerfile
+++ b/tile_gen/Dockerfile
@@ -1,5 +1,7 @@
FROM tiledb/tiledb-geospatial:latest
+WORKDIR /work
+
# TODO may need the unzip command to be installed if not already
RUN apt-get update && apt-get install -y unzip && \
wget http://ssec.wisc.edu/~rayg/pub/amqpfind.zip && \
diff --git a/tile_gen/generate_tiles.py b/tile_gen/generate_tiles.py
index 5fef7c8..f444e31 100644
--- a/tile_gen/generate_tiles.py
+++ b/tile_gen/generate_tiles.py
@@ -6,6 +6,7 @@ import warnings
import logging
import subprocess
import shutil
+from glob import glob
import tile_index
@@ -67,7 +68,9 @@ def main():
help="Product names to group together in each "
"'layer'. Product name must be in the filename.")
parser.add_argument('--shape-file', default='{product}.shp',
- help="Shapefile filename pattern to use and placed in the output directory. (default: '{product}.shp')")
+ help="Shapefile filename pattern to use and placed "
+ "in the output directory. "
+ "(default: '{product}.shp')")
parser.add_argument('out_dir',
help="Output path to save tile information to (ex. '/data/tiles/{product}')")
parser.add_argument('input_files', nargs="+",
@@ -88,8 +91,13 @@ def main():
# hardlink if needed
prod_files = list(link_or_copy(prod_files, out_dir))
+ # get all products in the current directory
+ ext = os.path.splitext(prod_files[-1])[-1]
+ all_prod_files = sorted(glob(os.path.join(out_dir, '*' + prod + '*' + ext)))
+
# create shape file
- tile_index.index(prod_files, shp_pathname)
+ LOG.info("Rebuilding shapefile index with:\n\t{}".format(", ".join(all_prod_files)))
+ tile_index.index(all_prod_files, shp_pathname)
diff --git a/tile_gen/run.sh b/tile_gen/run.sh
index dcd23bd..051058d 100755
--- a/tile_gen/run.sh
+++ b/tile_gen/run.sh
@@ -50,6 +50,8 @@ run_tile_gen() {
# generate_tiles.py will make a temporary directory
out_dir="${dst_dir}/tiles/${satellite_family}/${satellite_id}/${instrument}"
mkdir -p ${out_dir}
+ # add string formatting portion to separate add 'product' sub-directory
+ out_dir="${out_dir}/{product}"
python3 generate_tiles.py ${TILE_ARGS} -p ${G2G_PRODUCTS} -- ${out_dir} ${path}
# OUT/<product>/<product>.shp
glob_pattern="${out_dir}/*/*.shp"
--
GitLab