diff --git a/README.rst b/README.rst index 9dad9038f7f34bda06037591f07a6f0248877dbe..f029f1a1e5657d234ba722dd511bffb157cfe799 100644 --- a/README.rst +++ b/README.rst @@ -140,7 +140,7 @@ Backfill InfluxDB Database Insert data from an old tower file: - python -m aosstower.level_00.influxdb --influxdb-token <READ_WRITE_TOKEN> -vvv --bulk 5000 /data1/raw/aoss/tower/2018/05/08/aoss_tower.2018-05-08.ascii + python -m aosstower.level_00.influxdb --influxdb-token <READ_WRITE_TOKEN> -vvv --bulk 5000 /data/raw/aoss/tower/2018/05/08/aoss_tower.2018-05-08.ascii The above command sends data in blocks of 5000 records. This is to improve performance of sending data to the InfluxDB instead of sending one record @@ -159,7 +159,7 @@ use a large amount of memory. To insert a series of tower files: - find /data1/raw/mendota/buoy/2018/ -name "*.ascii" -print0 | sort -z | xargs -r0 -n1 python -m aosstower.level_00.influxdb -vvv --bulk 5000 --influxdb-token <READ_WRITE_TOKEN> + find /data/raw/mendota/buoy/2018/ -name "*.ascii" -print0 | sort -z | xargs -r0 -n1 python -m aosstower.level_00.influxdb -vvv --bulk 5000 --influxdb-token <READ_WRITE_TOKEN> The above command sorts the files by name which is important for the best performance. @@ -176,9 +176,9 @@ A bash one-liner for processing one year and running averages 1 month at a time .. code-block:: bash # AOSS Tower - for YEAR in `seq 2023 -1 2006`; do echo ${YEAR}; time find /data1/raw/aoss/tower/${YEAR} -name "*.ascii" -print0 | sort -z | xargs -r0 -n1 /opt/metobs/aoss_tower/bin/python -m aosstower.level_00.influxdb -v --sleep-interval 5.0 --bulk 10000 --influxdb-token $(</home/metobs/influxdbv2_metobs_rw.txt); for month in `seq -f %02.0f 1 12`; do /opt/metobs/aoss_tower/bin/python -m metobscommon.influxdb -vvv --influxdb-token $(</home/metobs/influxdbv2_metobs_rw.txt) run_manual_average --stations aoss.tower -s ${YEAR}-${month}-01T00:00:00 -e $(date -d "${YEAR}-${month}-01 + 1 month" +%Y-%m-%dT%H:%M:%S); sleep 5.0; done; echo "Done with $YEAR"; sleep 60; done + for YEAR in `seq 2023 -1 2006`; do echo ${YEAR}; time find /data/raw/aoss/tower/${YEAR} -name "*.ascii" -print0 | sort -z | xargs -r0 -n1 /opt/metobs/aoss_tower/bin/python -m aosstower.level_00.influxdb -v --sleep-interval 5.0 --bulk 10000 --influxdb-token $(</home/metobs/influxdbv2_metobs_rw.txt); for month in `seq -f %02.0f 1 12`; do /opt/metobs/aoss_tower/bin/python -m metobscommon.influxdb -vvv --influxdb-token $(</home/metobs/influxdbv2_metobs_rw.txt) run_manual_average --stations aoss.tower -s ${YEAR}-${month}-01T00:00:00 -e $(date -d "${YEAR}-${month}-01 + 1 month" +%Y-%m-%dT%H:%M:%S); sleep 5.0; done; echo "Done with $YEAR"; sleep 60; done # Mendota Buoy - for YEAR in `seq 2023 -1 2013`; do echo ${YEAR}; time find /data1/raw/mendota/buoy/${YEAR} \( -name "*limnodata.*ascii" -o -name "*metdata.*.ascii" \) -print0 | sort -z | xargs -r0 -n1 /opt/metobs/mendota_buoy/bin/python -m mendotabuoy.level_00.influxdb -v --sleep-interval 5.0 --bulk 10000 --influxdb-token $(</home/metobs/influxdbv2_metobs_rw.txt); for month in `seq -f %02.0f 2 11`; do /opt/metobs/mendota_buoy/bin/python -m metobscommon.influxdb -vvv --influxdb-token $(</home/metobs/influxdbv2_metobs_rw.txt) run_manual_average --stations mendota.buoy -s ${YEAR}-${month}-01T00:00:00 -e $(date -d "${YEAR}-${month}-01 + 1 month" +%Y-%m-%dT%H:%M:%S); sleep 5.0; done; echo "Done with $YEAR"; sleep 60; done + for YEAR in `seq 2023 -1 2013`; do echo ${YEAR}; time find /data/raw/mendota/buoy/${YEAR} \( -name "*limnodata.*ascii" -o -name "*metdata.*.ascii" \) -print0 | sort -z | xargs -r0 -n1 /opt/metobs/mendota_buoy/bin/python -m mendotabuoy.level_00.influxdb -v --sleep-interval 5.0 --bulk 10000 --influxdb-token $(</home/metobs/influxdbv2_metobs_rw.txt); for month in `seq -f %02.0f 2 11`; do /opt/metobs/mendota_buoy/bin/python -m metobscommon.influxdb -vvv --influxdb-token $(</home/metobs/influxdbv2_metobs_rw.txt) run_manual_average --stations mendota.buoy -s ${YEAR}-${month}-01T00:00:00 -e $(date -d "${YEAR}-${month}-01 + 1 month" +%Y-%m-%dT%H:%M:%S); sleep 5.0; done; echo "Done with $YEAR"; sleep 60; done diff --git a/metobscommon/archive/__init__.py b/metobscommon/archive/__init__.py index f9031f8c35ace2849d749e73c1853c59fbf5310e..664d697dedff8c78a9c9123b701f82eed4b29494 100644 --- a/metobscommon/archive/__init__.py +++ b/metobscommon/archive/__init__.py @@ -32,9 +32,9 @@ import logging from configparser import ConfigParser -INCOMING_ROOT = os.environ.get("METOBS_INCOMING", '/data1/incoming') -RAW_ROOT = os.environ.get("METOBS_RAW", '/data1/raw') -CACHE_ROOT = os.environ.get("METOBS_CACHE", '/data1/cache') +INCOMING_ROOT = os.environ.get("METOBS_INCOMING", '/data/incoming') +RAW_ROOT = os.environ.get("METOBS_RAW", '/data/raw') +CACHE_ROOT = os.environ.get("METOBS_CACHE", '/data/cache') DATA_LEVELS = [ "00", "01",