Skip to content
Snippets Groups Projects
Commit 2989315f authored by Yuan Gao's avatar Yuan Gao
Browse files

stable

parent 08817383
No related branches found
No related tags found
No related merge requests found
...@@ -6,24 +6,81 @@ import sys ...@@ -6,24 +6,81 @@ import sys
import os import os
import logging import logging
import pandas as pd import pandas as pd
import traceback
import subprocess
# Enable detailed CGI error reporting # Enable detailed CGI error reporting
cgitb.enable() cgitb.enable()
# Set up logging # Set up logging - use absolute path to ensure writability
LOG_FILE = "latency_viewer.log" LOG_DIR = "/var/www/html/web_internal/example/latency/logs"
os.makedirs(LOG_DIR, exist_ok=True)
LOG_FILE = os.path.join(LOG_DIR, "latency_viewer.log")
logging.basicConfig( logging.basicConfig(
level=logging.INFO, level=logging.DEBUG, # Changed to DEBUG for more detailed logs
format='%(asctime)s - %(levelname)s - %(message)s', format='%(asctime)s - %(levelname)s - %(message)s',
filename=LOG_FILE, filename=LOG_FILE,
filemode='a' filemode='a'
) )
logger = logging.getLogger() logger = logging.getLogger()
# Log script startup and environment information
logger.info("=" * 80)
logger.info("Data.py script starting")
logger.info(f"Current working directory: {os.getcwd()}")
logger.info(f"Script path: {os.path.abspath(__file__)}")
logger.info(f"Python version: {sys.version}")
logger.info(f"User running script: {os.getenv('USER') or 'Unknown'}")
try:
# Log system information
logger.info("System information:")
whoami_output = subprocess.check_output(["whoami"], stderr=subprocess.STDOUT).decode().strip()
logger.info(f"Current user from whoami: {whoami_output}")
# Check script permissions
logger.info(f"Script permissions: {oct(os.stat(__file__).st_mode)}")
# Check if we can write to tmp directory
tmp_test_path = "/tmp/data_py_test.txt"
try:
with open(tmp_test_path, 'w') as f:
f.write("Test")
os.remove(tmp_test_path)
logger.info("Successfully wrote to and removed test file in /tmp")
except Exception as e:
logger.error(f"Failed to write to /tmp: {str(e)}")
# Check if sudo is available
try:
sudo_test = subprocess.run(["sudo", "-l"], capture_output=True, text=True)
logger.info(f"Sudo permissions: {sudo_test.stdout}")
if sudo_test.returncode != 0:
logger.error(f"Sudo test failed: {sudo_test.stderr}")
except Exception as e:
logger.error(f"Error checking sudo permissions: {str(e)}")
except Exception as e:
logger.error(f"Error during environment checks: {str(e)}")
# Import functions from our shared module # Import functions from our shared module
# Adjust the path as needed to find the module # Adjust the path as needed to find the module
sys.path.append(os.path.dirname(os.path.abspath(__file__))) try:
from sat_db_functions import run_sat_latency_query, get_canonical_id, get_all_variants sys.path.append(os.path.dirname(os.path.abspath(__file__)))
logger.info(f"Looking for sat_db_functions in: {os.path.dirname(os.path.abspath(__file__))}")
# List directory contents to verify module existence
dir_contents = os.listdir(os.path.dirname(os.path.abspath(__file__)))
logger.info(f"Directory contents: {dir_contents}")
from sat_db_functions import run_sat_latency_query, get_canonical_id, get_all_variants
logger.info("Successfully imported from sat_db_functions")
except ImportError as e:
logger.error(f"Import error: {str(e)}")
logger.error(traceback.format_exc())
except Exception as e:
logger.error(f"Unexpected error during import: {str(e)}")
logger.error(traceback.format_exc())
def data_endpoint(): def data_endpoint():
""" """
...@@ -37,6 +94,8 @@ def data_endpoint(): ...@@ -37,6 +94,8 @@ def data_endpoint():
start_hour = form.getvalue("start_hour", "00:00") start_hour = form.getvalue("start_hour", "00:00")
end_hour = form.getvalue("end_hour", "23:59") end_hour = form.getvalue("end_hour", "23:59")
logger.info(f"Query parameters: start_date={start_date}, end_date={end_date}, start_hour={start_hour}, end_hour={end_hour}")
# Convert date and time to ISO format # Convert date and time to ISO format
start_datetime = f"{start_date}T{start_hour}:00" start_datetime = f"{start_date}T{start_hour}:00"
end_datetime = f"{end_date}T{end_hour}:59" end_datetime = f"{end_date}T{end_hour}:59"
...@@ -46,14 +105,19 @@ def data_endpoint(): ...@@ -46,14 +105,19 @@ def data_endpoint():
coverage = form.getvalue("coverage") coverage = form.getvalue("coverage")
instrument = form.getvalue("instrument") instrument = form.getvalue("instrument")
logger.info(f"Filter parameters: satellite_id={satellite_id}, coverage={coverage}, instrument={instrument}")
# Prepare filters # Prepare filters
filters = {} filters = {}
if satellite_id: if satellite_id:
# Get the canonical form # Get the canonical form
logger.info(f"Getting canonical form for: {satellite_id}")
canonical_id = get_canonical_id(satellite_id) canonical_id = get_canonical_id(satellite_id)
logger.info(f"Canonical ID: {canonical_id}")
# Get all variants of this canonical ID # Get all variants of this canonical ID
all_variants = get_all_variants(canonical_id) all_variants = get_all_variants(canonical_id)
logger.info(f"All variants: {all_variants}")
# Use all variants in the filter # Use all variants in the filter
filters["satellite-id"] = all_variants filters["satellite-id"] = all_variants
...@@ -69,22 +133,42 @@ def data_endpoint(): ...@@ -69,22 +133,42 @@ def data_endpoint():
logger.info(f"Data request - Period: {start_datetime} to {end_datetime}, Filters: {filters}") logger.info(f"Data request - Period: {start_datetime} to {end_datetime}, Filters: {filters}")
# Query the database # Query the database
data = run_sat_latency_query(start_datetime, end_datetime, filters) logger.info("About to call run_sat_latency_query...")
try:
data = run_sat_latency_query(start_datetime, end_datetime, filters)
logger.info(f"Query returned: {len(data) if data else 0} records")
except Exception as query_error:
logger.error(f"Error in run_sat_latency_query: {str(query_error)}")
logger.error(traceback.format_exc())
return {"message": f"Database query error: {str(query_error)}", "data": []}, 500
if not data: if not data:
logger.info("Query returned no data")
return {"message": "No data available for the selected period.", "data": []} return {"message": "No data available for the selected period.", "data": []}
# Convert to DataFrame for easier processing # Convert to DataFrame for easier processing
df = pd.DataFrame(data) logger.info("Converting to DataFrame...")
try:
df = pd.DataFrame(data)
logger.info(f"DataFrame created with shape: {df.shape}")
except Exception as df_error:
logger.error(f"Error creating DataFrame: {str(df_error)}")
logger.error(traceback.format_exc())
return {"message": f"Error creating DataFrame: {str(df_error)}", "data": []}, 500
# Clean and process data # Clean and process data
try: try:
logger.info("Processing DataFrame...")
# Normalize column names (case-insensitive matching) # Normalize column names (case-insensitive matching)
df.columns = [col.lower() for col in df.columns] df.columns = [col.lower() for col in df.columns]
logger.info(f"Columns after normalization: {list(df.columns)}")
# Clean latency data # Clean latency data
logger.info("Cleaning latency data...")
df['latency'] = pd.to_numeric(df['latency'], errors='coerce') df['latency'] = pd.to_numeric(df['latency'], errors='coerce')
df = df.dropna(subset=['latency']) df = df.dropna(subset=['latency'])
logger.info(f"DataFrame shape after cleaning: {df.shape}")
# Add missing columns with 'Not Available' default # Add missing columns with 'Not Available' default
default_columns = ['ingest_source', 'coverage', 'instrument', 'band', 'section', 'satellite_id'] default_columns = ['ingest_source', 'coverage', 'instrument', 'band', 'section', 'satellite_id']
...@@ -99,14 +183,18 @@ def data_endpoint(): ...@@ -99,14 +183,18 @@ def data_endpoint():
# Add canonical_satellite_id column # Add canonical_satellite_id column
if 'satellite_id' in df.columns: if 'satellite_id' in df.columns:
logger.info("Adding canonical_satellite_id column...")
df['canonical_satellite_id'] = df['satellite_id'].apply(get_canonical_id) df['canonical_satellite_id'] = df['satellite_id'].apply(get_canonical_id)
# Convert timestamps to string for JSON serialization # Convert timestamps to string for JSON serialization
if 'start_time' in df.columns: if 'start_time' in df.columns:
logger.info("Converting timestamps...")
df['start_time'] = pd.to_datetime(df['start_time']).astype(str) df['start_time'] = pd.to_datetime(df['start_time']).astype(str)
# Convert to records and handle NaN values # Convert to records and handle NaN values
logger.info("Converting to records...")
result = df.replace({pd.NA: "Not Available", pd.NaT: "Not Available"}).to_dict(orient="records") result = df.replace({pd.NA: "Not Available", pd.NaT: "Not Available"}).to_dict(orient="records")
logger.info(f"Created {len(result)} result records")
return { return {
"data": result, "data": result,
...@@ -118,12 +206,14 @@ def data_endpoint(): ...@@ -118,12 +206,14 @@ def data_endpoint():
} }
except Exception as e: except Exception as e:
logger.error(f"Error during data processing: {str(e)}", exc_info=True) logger.error(f"Error during data processing: {str(e)}")
return {"message": f"Data processing error: {str(e)}"}, 500 logger.error(traceback.format_exc())
return {"message": f"Data processing error: {str(e)}", "data": []}, 500
except Exception as e: except Exception as e:
logger.error(f"Error processing data request: {str(e)}", exc_info=True) logger.error(f"Error processing data request: {str(e)}")
return {"message": f"Internal Server Error: {str(e)}"}, 500 logger.error(traceback.format_exc())
return {"message": f"Internal Server Error: {str(e)}", "data": []}, 500
# Main entry point for CGI # Main entry point for CGI
if __name__ == "__main__": if __name__ == "__main__":
...@@ -131,12 +221,30 @@ if __name__ == "__main__": ...@@ -131,12 +221,30 @@ if __name__ == "__main__":
print("Content-Type: application/json") print("Content-Type: application/json")
print() # Empty line after headers print() # Empty line after headers
# Get the result from our endpoint function try:
result, status_code = data_endpoint() if isinstance(data_endpoint(), tuple) else (data_endpoint(), 200) # Get the result from our endpoint function
logger.info("Calling data_endpoint function...")
# If there's an error code, log it (CGI can't easily send HTTP status codes) result = data_endpoint()
if status_code != 200:
logger.warning(f"Returning error with status code {status_code}: {result}") # Handle tuple returns (for error responses)
if isinstance(result, tuple):
# Print JSON response response_data, status_code = result
print(json.dumps(result)) logger.warning(f"Returning error with status code {status_code}: {response_data}")
\ No newline at end of file else:
response_data, status_code = result, 200
# Print JSON response
logger.info(f"Returning response with status code {status_code} and {len(response_data.get('data', []))} records")
print(json.dumps(response_data))
except Exception as final_error:
logger.error(f"Final error in main block: {str(final_error)}")
logger.error(traceback.format_exc())
# Attempt to return a meaningful error
error_response = {
"error": "Critical error in script execution",
"message": str(final_error),
"data": []
}
print(json.dumps(error_response))
\ No newline at end of file
This diff is collapsed.
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment