Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
latency
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package Registry
Container Registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Yuan Gao
latency
Commits
2989315f
Commit
2989315f
authored
1 month ago
by
Yuan Gao
Browse files
Options
Downloads
Patches
Plain Diff
stable
parent
08817383
No related branches found
No related tags found
No related merge requests found
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
assets/python/data.py
+128
-20
128 additions, 20 deletions
assets/python/data.py
latency_viewer.log
+0
-196
0 additions, 196 deletions
latency_viewer.log
with
128 additions
and
216 deletions
assets/python/data.py
+
128
−
20
View file @
2989315f
...
...
@@ -6,24 +6,81 @@ import sys
import
os
import
logging
import
pandas
as
pd
import
traceback
import
subprocess
# Enable detailed CGI error reporting
cgitb
.
enable
()
# Set up logging
LOG_FILE
=
"
latency_viewer.log
"
# Set up logging - use absolute path to ensure writability
LOG_DIR
=
"
/var/www/html/web_internal/example/latency/logs
"
os
.
makedirs
(
LOG_DIR
,
exist_ok
=
True
)
LOG_FILE
=
os
.
path
.
join
(
LOG_DIR
,
"
latency_viewer.log
"
)
logging
.
basicConfig
(
level
=
logging
.
INFO
,
level
=
logging
.
DEBUG
,
# Changed to DEBUG for more detailed logs
format
=
'
%(asctime)s - %(levelname)s - %(message)s
'
,
filename
=
LOG_FILE
,
filemode
=
'
a
'
)
logger
=
logging
.
getLogger
()
# Log script startup and environment information
logger
.
info
(
"
=
"
*
80
)
logger
.
info
(
"
Data.py script starting
"
)
logger
.
info
(
f
"
Current working directory:
{
os
.
getcwd
()
}
"
)
logger
.
info
(
f
"
Script path:
{
os
.
path
.
abspath
(
__file__
)
}
"
)
logger
.
info
(
f
"
Python version:
{
sys
.
version
}
"
)
logger
.
info
(
f
"
User running script:
{
os
.
getenv
(
'
USER
'
)
or
'
Unknown
'
}
"
)
try
:
# Log system information
logger
.
info
(
"
System information:
"
)
whoami_output
=
subprocess
.
check_output
([
"
whoami
"
],
stderr
=
subprocess
.
STDOUT
).
decode
().
strip
()
logger
.
info
(
f
"
Current user from whoami:
{
whoami_output
}
"
)
# Check script permissions
logger
.
info
(
f
"
Script permissions:
{
oct
(
os
.
stat
(
__file__
).
st_mode
)
}
"
)
# Check if we can write to tmp directory
tmp_test_path
=
"
/tmp/data_py_test.txt
"
try
:
with
open
(
tmp_test_path
,
'
w
'
)
as
f
:
f
.
write
(
"
Test
"
)
os
.
remove
(
tmp_test_path
)
logger
.
info
(
"
Successfully wrote to and removed test file in /tmp
"
)
except
Exception
as
e
:
logger
.
error
(
f
"
Failed to write to /tmp:
{
str
(
e
)
}
"
)
# Check if sudo is available
try
:
sudo_test
=
subprocess
.
run
([
"
sudo
"
,
"
-l
"
],
capture_output
=
True
,
text
=
True
)
logger
.
info
(
f
"
Sudo permissions:
{
sudo_test
.
stdout
}
"
)
if
sudo_test
.
returncode
!=
0
:
logger
.
error
(
f
"
Sudo test failed:
{
sudo_test
.
stderr
}
"
)
except
Exception
as
e
:
logger
.
error
(
f
"
Error checking sudo permissions:
{
str
(
e
)
}
"
)
except
Exception
as
e
:
logger
.
error
(
f
"
Error during environment checks:
{
str
(
e
)
}
"
)
# Import functions from our shared module
# Adjust the path as needed to find the module
sys
.
path
.
append
(
os
.
path
.
dirname
(
os
.
path
.
abspath
(
__file__
)))
from
sat_db_functions
import
run_sat_latency_query
,
get_canonical_id
,
get_all_variants
try
:
sys
.
path
.
append
(
os
.
path
.
dirname
(
os
.
path
.
abspath
(
__file__
)))
logger
.
info
(
f
"
Looking for sat_db_functions in:
{
os
.
path
.
dirname
(
os
.
path
.
abspath
(
__file__
))
}
"
)
# List directory contents to verify module existence
dir_contents
=
os
.
listdir
(
os
.
path
.
dirname
(
os
.
path
.
abspath
(
__file__
)))
logger
.
info
(
f
"
Directory contents:
{
dir_contents
}
"
)
from
sat_db_functions
import
run_sat_latency_query
,
get_canonical_id
,
get_all_variants
logger
.
info
(
"
Successfully imported from sat_db_functions
"
)
except
ImportError
as
e
:
logger
.
error
(
f
"
Import error:
{
str
(
e
)
}
"
)
logger
.
error
(
traceback
.
format_exc
())
except
Exception
as
e
:
logger
.
error
(
f
"
Unexpected error during import:
{
str
(
e
)
}
"
)
logger
.
error
(
traceback
.
format_exc
())
def
data_endpoint
():
"""
...
...
@@ -37,6 +94,8 @@ def data_endpoint():
start_hour
=
form
.
getvalue
(
"
start_hour
"
,
"
00:00
"
)
end_hour
=
form
.
getvalue
(
"
end_hour
"
,
"
23:59
"
)
logger
.
info
(
f
"
Query parameters: start_date=
{
start_date
}
, end_date=
{
end_date
}
, start_hour=
{
start_hour
}
, end_hour=
{
end_hour
}
"
)
# Convert date and time to ISO format
start_datetime
=
f
"
{
start_date
}
T
{
start_hour
}
:00
"
end_datetime
=
f
"
{
end_date
}
T
{
end_hour
}
:59
"
...
...
@@ -46,14 +105,19 @@ def data_endpoint():
coverage
=
form
.
getvalue
(
"
coverage
"
)
instrument
=
form
.
getvalue
(
"
instrument
"
)
logger
.
info
(
f
"
Filter parameters: satellite_id=
{
satellite_id
}
, coverage=
{
coverage
}
, instrument=
{
instrument
}
"
)
# Prepare filters
filters
=
{}
if
satellite_id
:
# Get the canonical form
logger
.
info
(
f
"
Getting canonical form for:
{
satellite_id
}
"
)
canonical_id
=
get_canonical_id
(
satellite_id
)
logger
.
info
(
f
"
Canonical ID:
{
canonical_id
}
"
)
# Get all variants of this canonical ID
all_variants
=
get_all_variants
(
canonical_id
)
logger
.
info
(
f
"
All variants:
{
all_variants
}
"
)
# Use all variants in the filter
filters
[
"
satellite-id
"
]
=
all_variants
...
...
@@ -69,22 +133,42 @@ def data_endpoint():
logger
.
info
(
f
"
Data request - Period:
{
start_datetime
}
to
{
end_datetime
}
, Filters:
{
filters
}
"
)
# Query the database
data
=
run_sat_latency_query
(
start_datetime
,
end_datetime
,
filters
)
logger
.
info
(
"
About to call run_sat_latency_query...
"
)
try
:
data
=
run_sat_latency_query
(
start_datetime
,
end_datetime
,
filters
)
logger
.
info
(
f
"
Query returned:
{
len
(
data
)
if
data
else
0
}
records
"
)
except
Exception
as
query_error
:
logger
.
error
(
f
"
Error in run_sat_latency_query:
{
str
(
query_error
)
}
"
)
logger
.
error
(
traceback
.
format_exc
())
return
{
"
message
"
:
f
"
Database query error:
{
str
(
query_error
)
}
"
,
"
data
"
:
[]},
500
if
not
data
:
logger
.
info
(
"
Query returned no data
"
)
return
{
"
message
"
:
"
No data available for the selected period.
"
,
"
data
"
:
[]}
# Convert to DataFrame for easier processing
df
=
pd
.
DataFrame
(
data
)
logger
.
info
(
"
Converting to DataFrame...
"
)
try
:
df
=
pd
.
DataFrame
(
data
)
logger
.
info
(
f
"
DataFrame created with shape:
{
df
.
shape
}
"
)
except
Exception
as
df_error
:
logger
.
error
(
f
"
Error creating DataFrame:
{
str
(
df_error
)
}
"
)
logger
.
error
(
traceback
.
format_exc
())
return
{
"
message
"
:
f
"
Error creating DataFrame:
{
str
(
df_error
)
}
"
,
"
data
"
:
[]},
500
# Clean and process data
try
:
logger
.
info
(
"
Processing DataFrame...
"
)
# Normalize column names (case-insensitive matching)
df
.
columns
=
[
col
.
lower
()
for
col
in
df
.
columns
]
logger
.
info
(
f
"
Columns after normalization:
{
list
(
df
.
columns
)
}
"
)
# Clean latency data
logger
.
info
(
"
Cleaning latency data...
"
)
df
[
'
latency
'
]
=
pd
.
to_numeric
(
df
[
'
latency
'
],
errors
=
'
coerce
'
)
df
=
df
.
dropna
(
subset
=
[
'
latency
'
])
logger
.
info
(
f
"
DataFrame shape after cleaning:
{
df
.
shape
}
"
)
# Add missing columns with 'Not Available' default
default_columns
=
[
'
ingest_source
'
,
'
coverage
'
,
'
instrument
'
,
'
band
'
,
'
section
'
,
'
satellite_id
'
]
...
...
@@ -99,14 +183,18 @@ def data_endpoint():
# Add canonical_satellite_id column
if
'
satellite_id
'
in
df
.
columns
:
logger
.
info
(
"
Adding canonical_satellite_id column...
"
)
df
[
'
canonical_satellite_id
'
]
=
df
[
'
satellite_id
'
].
apply
(
get_canonical_id
)
# Convert timestamps to string for JSON serialization
if
'
start_time
'
in
df
.
columns
:
logger
.
info
(
"
Converting timestamps...
"
)
df
[
'
start_time
'
]
=
pd
.
to_datetime
(
df
[
'
start_time
'
]).
astype
(
str
)
# Convert to records and handle NaN values
logger
.
info
(
"
Converting to records...
"
)
result
=
df
.
replace
({
pd
.
NA
:
"
Not Available
"
,
pd
.
NaT
:
"
Not Available
"
}).
to_dict
(
orient
=
"
records
"
)
logger
.
info
(
f
"
Created
{
len
(
result
)
}
result records
"
)
return
{
"
data
"
:
result
,
...
...
@@ -118,12 +206,14 @@ def data_endpoint():
}
except
Exception
as
e
:
logger
.
error
(
f
"
Error during data processing:
{
str
(
e
)
}
"
,
exc_info
=
True
)
return
{
"
message
"
:
f
"
Data processing error:
{
str
(
e
)
}
"
},
500
logger
.
error
(
f
"
Error during data processing:
{
str
(
e
)
}
"
)
logger
.
error
(
traceback
.
format_exc
())
return
{
"
message
"
:
f
"
Data processing error:
{
str
(
e
)
}
"
,
"
data
"
:
[]},
500
except
Exception
as
e
:
logger
.
error
(
f
"
Error processing data request:
{
str
(
e
)
}
"
,
exc_info
=
True
)
return
{
"
message
"
:
f
"
Internal Server Error:
{
str
(
e
)
}
"
},
500
logger
.
error
(
f
"
Error processing data request:
{
str
(
e
)
}
"
)
logger
.
error
(
traceback
.
format_exc
())
return
{
"
message
"
:
f
"
Internal Server Error:
{
str
(
e
)
}
"
,
"
data
"
:
[]},
500
# Main entry point for CGI
if
__name__
==
"
__main__
"
:
...
...
@@ -131,12 +221,30 @@ if __name__ == "__main__":
print
(
"
Content-Type: application/json
"
)
print
()
# Empty line after headers
# Get the result from our endpoint function
result
,
status_code
=
data_endpoint
()
if
isinstance
(
data_endpoint
(),
tuple
)
else
(
data_endpoint
(),
200
)
# If there's an error code, log it (CGI can't easily send HTTP status codes)
if
status_code
!=
200
:
logger
.
warning
(
f
"
Returning error with status code
{
status_code
}
:
{
result
}
"
)
# Print JSON response
print
(
json
.
dumps
(
result
))
\ No newline at end of file
try
:
# Get the result from our endpoint function
logger
.
info
(
"
Calling data_endpoint function...
"
)
result
=
data_endpoint
()
# Handle tuple returns (for error responses)
if
isinstance
(
result
,
tuple
):
response_data
,
status_code
=
result
logger
.
warning
(
f
"
Returning error with status code
{
status_code
}
:
{
response_data
}
"
)
else
:
response_data
,
status_code
=
result
,
200
# Print JSON response
logger
.
info
(
f
"
Returning response with status code
{
status_code
}
and
{
len
(
response_data
.
get
(
'
data
'
,
[]))
}
records
"
)
print
(
json
.
dumps
(
response_data
))
except
Exception
as
final_error
:
logger
.
error
(
f
"
Final error in main block:
{
str
(
final_error
)
}
"
)
logger
.
error
(
traceback
.
format_exc
())
# Attempt to return a meaningful error
error_response
=
{
"
error
"
:
"
Critical error in script execution
"
,
"
message
"
:
str
(
final_error
),
"
data
"
:
[]
}
print
(
json
.
dumps
(
error_response
))
\ No newline at end of file
This diff is collapsed.
Click to expand it.
latency_viewer.log
+
0
−
196
View file @
2989315f
2025-03-07 16:50:02,658 - WARNING - Relationships file not found: satellite_relationships.json
2025-03-07 16:50:02,658 - WARNING - Relationships file not found: satellite_relationships.json
2025-03-07 16:51:47,864 - WARNING - Relationships file not found: satellite_relationships.json
2025-03-07 16:51:47,864 - WARNING - Relationships file not found: satellite_relationships.json
2025-03-07 16:51:52,682 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': '4B'}
2025-03-07 16:51:52,682 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "4B"
2025-03-07 16:51:52,682 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 16:51:57,885 - INFO - Command output (first 200 chars): [{"satellite_ID": "4B", "band": null, "coverage": null, "ingest_source": null, "instrument": "GIIRS", "section": null, "start_time": "2025-03-07T00:03:28+00:00", "latency": 776.71}, {"satellite_ID": "...
2025-03-07 16:51:57,885 - INFO - Successfully parsed JSON data: 77 records found
2025-03-07 16:51:57,982 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': '4B'}
2025-03-07 16:51:57,982 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "4B"
2025-03-07 16:51:57,982 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 16:52:01,160 - INFO - Command output (first 200 chars): [{"satellite_ID": "4B", "band": null, "coverage": null, "ingest_source": null, "instrument": "GIIRS", "section": null, "start_time": "2025-03-07T00:03:28+00:00", "latency": 776.71}, {"satellite_ID": "...
2025-03-07 16:52:01,161 - INFO - Successfully parsed JSON data: 77 records found
2025-03-07 16:52:08,507 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': 'composite'}
2025-03-07 16:52:08,507 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "composite"
2025-03-07 16:52:08,507 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 16:52:11,763 - WARNING - Command returned empty output
2025-03-07 16:52:11,763 - ERROR - Failed to parse JSON output: Expecting value: line 1 column 1 (char 0)
2025-03-07 16:52:11,763 - ERROR - Raw output (first 500 chars): ...
2025-03-07 16:52:11,763 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': 'composite'}
2025-03-07 16:52:11,763 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "composite"
2025-03-07 16:52:11,764 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 16:52:14,972 - WARNING - Command returned empty output
2025-03-07 16:52:14,972 - ERROR - Failed to parse JSON output: Expecting value: line 1 column 1 (char 0)
2025-03-07 16:52:14,972 - ERROR - Raw output (first 500 chars): ...
2025-03-07 16:52:57,421 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': 'G16'}
2025-03-07 16:52:57,422 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "G16" "g16"
2025-03-07 16:52:57,422 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 16:53:00,726 - INFO - Command output (first 200 chars): [{"satellite_ID": "G16", "band": "Not Available", "coverage": null, "ingest_source": "inge GLM GRB-R v1.0.0 : grbdelta.ssec.wisc.edu", "instrument": "GLM", "section": null, "start_time": "2025-03-07T0...
2025-03-07 16:53:00,745 - INFO - Successfully parsed JSON data: 8668 records found
2025-03-07 16:53:00,918 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': 'G16'}
2025-03-07 16:53:00,918 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "G16" "g16"
2025-03-07 16:53:00,918 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 16:53:04,131 - INFO - Command output (first 200 chars): [{"satellite_ID": "G16", "band": "Not Available", "coverage": null, "ingest_source": "inge GLM GRB-R v1.0.0 : grbdelta.ssec.wisc.edu", "instrument": "GLM", "section": null, "start_time": "2025-03-07T0...
2025-03-07 16:53:04,149 - INFO - Successfully parsed JSON data: 8668 records found
2025-03-07 17:02:04,459 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': '4B', 'instrument': 'GIIRS'}
2025-03-07 17:02:04,459 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "4B" --instrument "GIIRS"
2025-03-07 17:02:04,459 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 17:02:07,843 - INFO - Command output (first 200 chars): [{"satellite_ID": "4B", "band": null, "coverage": null, "ingest_source": null, "instrument": "GIIRS", "section": null, "start_time": "2025-03-07T00:03:28+00:00", "latency": 776.71}, {"satellite_ID": "...
2025-03-07 17:02:07,844 - INFO - Successfully parsed JSON data: 77 records found
2025-03-07 17:02:07,863 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': '4B', 'instrument': 'GIIRS'}
2025-03-07 17:02:07,863 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "4B" --instrument "GIIRS"
2025-03-07 17:02:07,863 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 17:02:10,832 - INFO - Command output (first 200 chars): [{"satellite_ID": "4B", "band": null, "coverage": null, "ingest_source": null, "instrument": "GIIRS", "section": null, "start_time": "2025-03-07T00:03:28+00:00", "latency": 776.71}, {"satellite_ID": "...
2025-03-07 17:02:10,832 - INFO - Successfully parsed JSON data: 77 records found
2025-03-07 17:02:15,169 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': 'G16', 'instrument': 'ABI'}
2025-03-07 17:02:15,169 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "G16" "g16" --instrument "ABI"
2025-03-07 17:02:15,170 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 17:02:18,196 - INFO - Command output (first 200 chars): [{"satellite_ID": "G16", "band": "1", "coverage": "Mesoscale-1", "ingest_source": "inge ABI GRB-R v1.0.0 : grbdelta.ssec.wisc.edu", "instrument": "ABI", "section": null, "start_time": "2025-03-07T00:0...
2025-03-07 17:02:18,211 - INFO - Successfully parsed JSON data: 7311 records found
2025-03-07 17:02:18,369 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': 'G16', 'instrument': 'ABI'}
2025-03-07 17:02:18,369 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "G16" "g16" --instrument "ABI"
2025-03-07 17:02:18,369 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 17:02:21,397 - INFO - Command output (first 200 chars): [{"satellite_ID": "G16", "band": "1", "coverage": "Mesoscale-1", "ingest_source": "inge ABI GRB-R v1.0.0 : grbdelta.ssec.wisc.edu", "instrument": "ABI", "section": null, "start_time": "2025-03-07T00:0...
2025-03-07 17:02:21,411 - INFO - Successfully parsed JSON data: 7311 records found
2025-03-07 17:06:17,380 - INFO - Expanded satellite ID 4B to variants: ['4B']
2025-03-07 17:06:17,380 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': ['4B'], 'instrument': 'GIIRS'}
2025-03-07 17:06:17,380 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "4B" --instrument "GIIRS"
2025-03-07 17:06:17,381 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 17:06:20,651 - INFO - Command output (first 200 chars): [{"satellite_ID": "4B", "band": null, "coverage": null, "ingest_source": null, "instrument": "GIIRS", "section": null, "start_time": "2025-03-07T00:03:28+00:00", "latency": 776.71}, {"satellite_ID": "...
2025-03-07 17:06:20,651 - INFO - Successfully parsed JSON data: 77 records found
2025-03-07 17:06:20,671 - INFO - Expanded satellite ID 4B to variants: ['4B']
2025-03-07 17:06:20,671 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': ['4B'], 'instrument': 'GIIRS'}
2025-03-07 17:06:20,671 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "4B" --instrument "GIIRS"
2025-03-07 17:06:20,671 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 17:06:23,696 - INFO - Command output (first 200 chars): [{"satellite_ID": "4B", "band": null, "coverage": null, "ingest_source": null, "instrument": "GIIRS", "section": null, "start_time": "2025-03-07T00:03:28+00:00", "latency": 776.71}, {"satellite_ID": "...
2025-03-07 17:06:23,697 - INFO - Successfully parsed JSON data: 77 records found
2025-03-07 17:06:34,484 - INFO - Expanded satellite ID G16 to variants: ['G16', 'g16']
2025-03-07 17:06:34,484 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': ['G16', 'g16'], 'instrument': 'ABI'}
2025-03-07 17:06:34,484 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "G16" "g16" --instrument "ABI"
2025-03-07 17:06:34,484 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 17:06:37,804 - INFO - Command output (first 200 chars): [{"satellite_ID": "G16", "band": "1", "coverage": "Mesoscale-1", "ingest_source": "inge ABI GRB-R v1.0.0 : grbdelta.ssec.wisc.edu", "instrument": "ABI", "section": null, "start_time": "2025-03-07T00:0...
2025-03-07 17:06:37,819 - INFO - Successfully parsed JSON data: 7311 records found
2025-03-07 17:06:37,962 - INFO - Expanded satellite ID G16 to variants: ['G16', 'g16']
2025-03-07 17:06:37,962 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': ['G16', 'g16'], 'instrument': 'ABI'}
2025-03-07 17:06:37,962 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "G16" "g16" --instrument "ABI"
2025-03-07 17:06:37,962 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 17:06:41,024 - INFO - Command output (first 200 chars): [{"satellite_ID": "G16", "band": "1", "coverage": "Mesoscale-1", "ingest_source": "inge ABI GRB-R v1.0.0 : grbdelta.ssec.wisc.edu", "instrument": "ABI", "section": null, "start_time": "2025-03-07T00:0...
2025-03-07 17:06:41,039 - INFO - Successfully parsed JSON data: 7311 records found
2025-03-07 17:07:56,791 - INFO - Expanded satellite ID NOAA-20 to variants: ['NOAA-20', 'n20']
2025-03-07 17:07:56,791 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': ['NOAA-20', 'n20'], 'instrument': 'VIIRS'}
2025-03-07 17:07:56,791 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "NOAA-20" "n20" --instrument "VIIRS"
2025-03-07 17:07:56,791 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 17:08:00,425 - WARNING - Command returned empty output
2025-03-07 17:08:00,425 - ERROR - Failed to parse JSON output: Expecting value: line 1 column 1 (char 0)
2025-03-07 17:08:00,425 - ERROR - Raw output (first 500 chars): ...
2025-03-07 17:08:00,426 - INFO - Expanded satellite ID NOAA-20 to variants: ['NOAA-20', 'n20']
2025-03-07 17:08:00,426 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': ['NOAA-20', 'n20'], 'instrument': 'VIIRS'}
2025-03-07 17:08:00,426 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "NOAA-20" "n20" --instrument "VIIRS"
2025-03-07 17:08:00,426 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 17:08:03,402 - WARNING - Command returned empty output
2025-03-07 17:08:03,402 - ERROR - Failed to parse JSON output: Expecting value: line 1 column 1 (char 0)
2025-03-07 17:08:03,402 - ERROR - Raw output (first 500 chars): ...
2025-03-07 17:08:27,679 - INFO - Expanded satellite ID NOAA-19 to variants: ['NOAA-19', 'n19']
2025-03-07 17:08:27,679 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': ['NOAA-19', 'n19'], 'instrument': 'amsu'}
2025-03-07 17:08:27,679 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "NOAA-19" "n19" --instrument "amsu"
2025-03-07 17:08:27,679 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 17:08:30,979 - WARNING - Command returned empty output
2025-03-07 17:08:30,979 - ERROR - Failed to parse JSON output: Expecting value: line 1 column 1 (char 0)
2025-03-07 17:08:30,979 - ERROR - Raw output (first 500 chars): ...
2025-03-07 17:08:30,979 - INFO - Expanded satellite ID NOAA-19 to variants: ['NOAA-19', 'n19']
2025-03-07 17:08:30,979 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': ['NOAA-19', 'n19'], 'instrument': 'amsu'}
2025-03-07 17:08:30,979 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "NOAA-19" "n19" --instrument "amsu"
2025-03-07 17:08:30,980 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 17:08:34,271 - WARNING - Command returned empty output
2025-03-07 17:08:34,271 - ERROR - Failed to parse JSON output: Expecting value: line 1 column 1 (char 0)
2025-03-07 17:08:34,271 - ERROR - Raw output (first 500 chars): ...
2025-03-07 17:09:42,447 - INFO - Expanded satellite ID NOAA-18 to variants: ['NOAA-18']
2025-03-07 17:09:42,447 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': ['NOAA-18'], 'instrument': 'amsu'}
2025-03-07 17:09:42,447 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "NOAA-18" --instrument "amsu"
2025-03-07 17:09:42,447 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 17:09:45,528 - WARNING - Command returned empty output
2025-03-07 17:09:45,528 - ERROR - Failed to parse JSON output: Expecting value: line 1 column 1 (char 0)
2025-03-07 17:09:45,528 - ERROR - Raw output (first 500 chars): ...
2025-03-07 17:09:45,529 - INFO - Expanded satellite ID NOAA-18 to variants: ['NOAA-18']
2025-03-07 17:09:45,529 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': ['NOAA-18'], 'instrument': 'amsu'}
2025-03-07 17:09:45,529 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "NOAA-18" --instrument "amsu"
2025-03-07 17:09:45,529 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 17:09:48,515 - WARNING - Command returned empty output
2025-03-07 17:09:48,515 - ERROR - Failed to parse JSON output: Expecting value: line 1 column 1 (char 0)
2025-03-07 17:09:48,515 - ERROR - Raw output (first 500 chars): ...
2025-03-07 17:11:02,269 - INFO - Expanded satellite ID NOAA-15 to variants: ['NOAA-15']
2025-03-07 17:11:02,269 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': ['NOAA-15'], 'instrument': 'amsu'}
2025-03-07 17:11:02,269 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "NOAA-15" --instrument "amsu"
2025-03-07 17:11:02,269 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 17:11:05,306 - WARNING - Command returned empty output
2025-03-07 17:11:05,306 - ERROR - Failed to parse JSON output: Expecting value: line 1 column 1 (char 0)
2025-03-07 17:11:05,306 - ERROR - Raw output (first 500 chars): ...
2025-03-07 17:11:05,306 - INFO - Expanded satellite ID NOAA-15 to variants: ['NOAA-15']
2025-03-07 17:11:05,307 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': ['NOAA-15'], 'instrument': 'amsu'}
2025-03-07 17:11:05,307 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "NOAA-15" --instrument "amsu"
2025-03-07 17:11:05,307 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 17:11:08,218 - WARNING - Command returned empty output
2025-03-07 17:11:08,218 - ERROR - Failed to parse JSON output: Expecting value: line 1 column 1 (char 0)
2025-03-07 17:11:08,219 - ERROR - Raw output (first 500 chars): ...
2025-03-07 17:11:37,274 - INFO - Expanded satellite ID NOAA-20 to variants: ['NOAA-20', 'n20']
2025-03-07 17:11:37,274 - INFO - Data request - Period: 2025-03-06T00:00:00 to 2025-03-06T23:59:59, Filters: {'satellite-id': ['NOAA-20', 'n20'], 'instrument': 'VIIRS'}
2025-03-07 17:11:37,274 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-06T00:00:00' --until '2025-03-06T23:59:59' --output-type json --satellite-id "NOAA-20" "n20" --instrument "VIIRS"
2025-03-07 17:11:37,274 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 17:11:41,744 - INFO - Command output (first 200 chars): [{"satellite_ID": "n20", "band": null, "coverage": null, "ingest_source": null, "instrument": "VIIRS", "section": null, "start_time": "2025-03-06T01:28:05+00:00", "latency": 8949.698}, {"satellite_ID"...
2025-03-07 17:11:41,744 - INFO - Successfully parsed JSON data: 22 records found
2025-03-07 17:11:41,762 - INFO - Expanded satellite ID NOAA-20 to variants: ['NOAA-20', 'n20']
2025-03-07 17:11:41,762 - INFO - Data request - Period: 2025-03-06T00:00:00 to 2025-03-06T23:59:59, Filters: {'satellite-id': ['NOAA-20', 'n20'], 'instrument': 'VIIRS'}
2025-03-07 17:11:41,762 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-06T00:00:00' --until '2025-03-06T23:59:59' --output-type json --satellite-id "NOAA-20" "n20" --instrument "VIIRS"
2025-03-07 17:11:41,762 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 17:11:45,033 - INFO - Command output (first 200 chars): [{"satellite_ID": "n20", "band": null, "coverage": null, "ingest_source": null, "instrument": "VIIRS", "section": null, "start_time": "2025-03-06T01:28:05+00:00", "latency": 8949.698}, {"satellite_ID"...
2025-03-07 17:11:45,033 - INFO - Successfully parsed JSON data: 22 records found
2025-03-07 17:21:25,901 - INFO - Expanded satellite ID composite to variants: ['composite']
2025-03-07 17:21:25,901 - INFO - Data request - Period: 2025-03-08T00:00:00 to 2025-03-08T23:59:59, Filters: {'satellite-id': ['composite'], 'instrument': 'VIIRS'}
2025-03-07 17:21:25,901 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-08T00:00:00' --until '2025-03-08T23:59:59' --output-type json --satellite-id "composite" --instrument "VIIRS"
2025-03-07 17:21:25,902 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 17:21:28,954 - ERROR - Command failed with exit code 1: Traceback (most recent call last):
File "/home/oper/.mdrexler_conda/bin/sat_latency_interface", line 8, in <module>
sys.exit(main())
File "/home/oper/.mdrexler_conda/lib/python3.9/site-packages/sat_latency/_utils.py", line 50, in wrapper
return function(*args, **kwargs)
File "/home/oper/.mdrexler_conda/lib/python3.9/site-packages/sat_latency/interface.py", line 285, in main
df = satellite_data_from_filters(
File "/home/oper/.mdrexler_conda/lib/python3.9/site-packages/sat_latency/interface.py", line 266, in satellite_data_from_filters
df = read_satellite_data(
File "/home/oper/.mdrexler_conda/lib/python3.9/site-packages/sat_latency/pipeline/load.py", line 101, in read_satellite_data
tbl = pa.Table.from_batches(
File "pyarrow/table.pxi", line 4760, in pyarrow.lib.Table.from_batches
ValueError: Must pass schema, or at least one RecordBatch
2025-03-07 17:21:28,955 - INFO - Expanded satellite ID composite to variants: ['composite']
2025-03-07 17:21:28,955 - INFO - Data request - Period: 2025-03-08T00:00:00 to 2025-03-08T23:59:59, Filters: {'satellite-id': ['composite'], 'instrument': 'VIIRS'}
2025-03-07 17:21:28,955 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-08T00:00:00' --until '2025-03-08T23:59:59' --output-type json --satellite-id "composite" --instrument "VIIRS"
2025-03-07 17:21:28,955 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 17:21:31,907 - ERROR - Command failed with exit code 1: Traceback (most recent call last):
File "/home/oper/.mdrexler_conda/bin/sat_latency_interface", line 8, in <module>
sys.exit(main())
File "/home/oper/.mdrexler_conda/lib/python3.9/site-packages/sat_latency/_utils.py", line 50, in wrapper
return function(*args, **kwargs)
File "/home/oper/.mdrexler_conda/lib/python3.9/site-packages/sat_latency/interface.py", line 285, in main
df = satellite_data_from_filters(
File "/home/oper/.mdrexler_conda/lib/python3.9/site-packages/sat_latency/interface.py", line 266, in satellite_data_from_filters
df = read_satellite_data(
File "/home/oper/.mdrexler_conda/lib/python3.9/site-packages/sat_latency/pipeline/load.py", line 101, in read_satellite_data
tbl = pa.Table.from_batches(
File "pyarrow/table.pxi", line 4760, in pyarrow.lib.Table.from_batches
ValueError: Must pass schema, or at least one RecordBatch
2025-03-07 17:41:18,389 - INFO - Expanded satellite ID 4B to variants: ['4B']
2025-03-07 17:41:18,389 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': ['4B'], 'instrument': 'GIIRS'}
2025-03-07 17:41:18,389 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "4B" --instrument "GIIRS"
2025-03-07 17:41:18,389 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 17:41:21,503 - INFO - Command output (first 200 chars): [{"satellite_ID": "4B", "band": null, "coverage": null, "ingest_source": null, "instrument": "GIIRS", "section": null, "start_time": "2025-03-07T00:03:28+00:00", "latency": 776.71}, {"satellite_ID": "...
2025-03-07 17:41:21,503 - INFO - Successfully parsed JSON data: 77 records found
2025-03-07 17:41:21,522 - INFO - Expanded satellite ID 4B to variants: ['4B']
2025-03-07 17:41:21,522 - INFO - Data request - Period: 2025-03-07T00:00:00 to 2025-03-07T23:59:59, Filters: {'satellite-id': ['4B'], 'instrument': 'GIIRS'}
2025-03-07 17:41:21,522 - INFO - Running command: module load miniconda/3.6-base && source activate ~/.mdrexler_conda && sat_latency_interface -d /data/sat_latency --from '2025-03-07T00:00:00' --until '2025-03-07T23:59:59' --output-type json --satellite-id "4B" --instrument "GIIRS"
2025-03-07 17:41:21,522 - INFO - Executing: sudo -u oper -i /tmp/run_sat_latency.sh
2025-03-07 17:41:24,440 - INFO - Command output (first 200 chars): [{"satellite_ID": "4B", "band": null, "coverage": null, "ingest_source": null, "instrument": "GIIRS", "section": null, "start_time": "2025-03-07T00:03:28+00:00", "latency": 776.71}, {"satellite_ID": "...
2025-03-07 17:41:24,441 - INFO - Successfully parsed JSON data: 77 records found
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment
