ingest.sh 1.69 KB
Newer Older
1
#!/usr/bin/env bash
2
3
4
# TODO: do we want to keep files even if processed?
#       if so, put them in quarantine right away and make a hard link @ processing dir for recognized files
 
5
6
7
set -u
set -e

8
9
10
DIR="${BASH_SOURCE%/*}"
if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
. "$DIR/consts.sh"
Nick Bearson's avatar
Nick Bearson committed
11

12
13
LOG=${LOGS}/ingest.log
exec >> $LOG 2>&1
14
15
16
17

filepath=$(readlink -f $1)
filename=$(basename $filepath)

18
19
recognized="yes"

20
21
# set $SCRIPT based on input type
if [ ${filename:0:2} == "LC" ] && [ ${filename: -4} == ".zip" ]; then  # landsat zip files
22
  log "SAW LANDSAT ZIP: ${filename}"
23
24
  SCRIPT="${SRC}/process-LCzip.sh"
  LOG_TO="${LOGS}/process-LCzip.log"
Nick Bearson's avatar
Nick Bearson committed
25
elif [ ${filename:0:2} == "LC" ] && [ ${filename: -7} == ".tar.gz" ]; then  # landsat zip files
26
  log "SAW LANDSAT TAR.GZ: ${filename}"
27
28
  SCRIPT="${SRC}/process-LCtgz.sh"
  LOG_TO="${LOGS}/process-LCtgz.log"
29
else # not recognized
30
31
  log "SAW (NOT RECOGNIZED): ${filename}"
  recognized="idk"
32
33
fi

34
35
36
# just quarantine everything for now in case we want it later
# (is quarantine the right word anymore? oh well...)
mkdir -p ${QUARANTINE}
37
mv -f $filepath ${QUARANTINE}/${filename} # overwrite so we don't crash if reprocessing
38

39
40
41
42
if [ "${recognized}" == "idk" ]; then
  exit 0
fi

43
# set up processing directory
44
45
mkdir -p ${PROCESSING}/${filename}
cd ${PROCESSING}/${filename}
46
47

# link input file in and go
48
49
ln ${QUARANTINE}/${filename} ${PROCESSING}/${filename}/${filename}  # make a hard link so we don't actually duplicate input data; OK as long as processing doesn't mess w/ it
$SCRIPT ${PROCESSING}/${filename}/${filename} &>> ${LOG_TO}
50
51
if [ $? -eq 0 ]; then
  rm -r ${PROCESSING}/${filename}
52
else
53
  log "ERROR [$?] while processing ${PROCESSING}/${filename}" # we shouldn't hit this due to `set -e`
54
fi