Fix QRZ download runaway process with timeouts and safety limits
这个提交包含在:
当前提交
f25f4b569d
共有 2 个文件被更改,包括 50 次插入 和 14 次删除
|
|
@ -336,11 +336,32 @@ class Qrz extends CI_Controller {
|
||||||
curl_setopt( $ch, CURLOPT_FOLLOWLOCATION, 1); // Okay
|
curl_setopt( $ch, CURLOPT_FOLLOWLOCATION, 1); // Okay
|
||||||
curl_setopt( $ch, CURLOPT_HEADER, 0); // Correct - don't need response headers
|
curl_setopt( $ch, CURLOPT_HEADER, 0); // Correct - don't need response headers
|
||||||
curl_setopt( $ch, CURLOPT_RETURNTRANSFER, true); // Correct - get response as string
|
curl_setopt( $ch, CURLOPT_RETURNTRANSFER, true); // Correct - get response as string
|
||||||
|
curl_setopt( $ch, CURLOPT_TIMEOUT, 300); // 5 minute timeout
|
||||||
|
curl_setopt( $ch, CURLOPT_CONNECTTIMEOUT, 30); // 30 second connection timeout
|
||||||
|
|
||||||
$content = curl_exec($ch); // Get raw content
|
$content = curl_exec($ch); // Get raw content
|
||||||
$curl_error = curl_error($ch); // Check for cURL errors
|
$curl_error = curl_error($ch); // Check for cURL errors
|
||||||
|
$http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE); // Get HTTP response code
|
||||||
curl_close($ch);
|
curl_close($ch);
|
||||||
|
|
||||||
|
if ($curl_error) { // Check for cURL level errors first
|
||||||
|
$error_message = "QRZ download cURL error: " . $curl_error;
|
||||||
|
log_message('error', $error_message . ' API Key used: ' . $qrz_api_key);
|
||||||
|
return ['status' => 'error', 'message' => $error_message];
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($http_code !== 200) {
|
||||||
|
$error_message = "QRZ download HTTP error: HTTP " . $http_code;
|
||||||
|
log_message('error', $error_message . ' API Key used: ' . $qrz_api_key);
|
||||||
|
return ['status' => 'error', 'message' => $error_message];
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($content === false || $content === '') { // Check if curl_exec failed or returned empty
|
||||||
|
$error_message = "QRZ download failed: No content received from QRZ.com.";
|
||||||
|
log_message('error', $error_message . ' API Key used: ' . $qrz_api_key);
|
||||||
|
return ['status' => 'error', 'message' => $error_message];
|
||||||
|
}
|
||||||
|
|
||||||
// Find the start of the ADIF data after "ADIF="
|
// Find the start of the ADIF data after "ADIF="
|
||||||
$adif_start_pos = strpos($content, 'ADIF=');
|
$adif_start_pos = strpos($content, 'ADIF=');
|
||||||
if ($adif_start_pos !== false) {
|
if ($adif_start_pos !== false) {
|
||||||
|
|
@ -388,18 +409,6 @@ class Qrz extends CI_Controller {
|
||||||
$content = substr($content, 0, $truncate_pos);
|
$content = substr($content, 0, $truncate_pos);
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($curl_error) { // Check for cURL level errors first
|
|
||||||
$error_message = "QRZ download cURL error: " . $curl_error;
|
|
||||||
log_message('error', $error_message . ' API Key used: ' . $qrz_api_key);
|
|
||||||
return ['status' => 'error', 'message' => $error_message];
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($content === false || $content === '') { // Check if curl_exec failed or returned empty
|
|
||||||
$error_message = "QRZ download failed: No content received from QRZ.com.";
|
|
||||||
log_message('error', $error_message . ' API Key used: ' . $qrz_api_key);
|
|
||||||
return ['status' => 'error', 'message' => $error_message];
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check for QRZ API specific error messages
|
// Check for QRZ API specific error messages
|
||||||
if (strpos($content, 'STATUS=FAIL') !== false || strpos($content, 'STATUS=AUTH') !== false) {
|
if (strpos($content, 'STATUS=FAIL') !== false || strpos($content, 'STATUS=AUTH') !== false) {
|
||||||
// Extract reason if possible, otherwise use full content
|
// Extract reason if possible, otherwise use full content
|
||||||
|
|
@ -446,7 +455,7 @@ class Qrz extends CI_Controller {
|
||||||
$config['qrz_rcvd_mark'] = 'Y';
|
$config['qrz_rcvd_mark'] = 'Y';
|
||||||
|
|
||||||
ini_set('memory_limit', '-1');
|
ini_set('memory_limit', '-1');
|
||||||
set_time_limit(0);
|
set_time_limit(1800); // 30 minutes max execution time instead of unlimited
|
||||||
|
|
||||||
$this->load->library('adif_parser');
|
$this->load->library('adif_parser');
|
||||||
|
|
||||||
|
|
@ -475,8 +484,24 @@ class Qrz extends CI_Controller {
|
||||||
$batch_data = [];
|
$batch_data = [];
|
||||||
$batch_size = 500; // Process 500 records at a time
|
$batch_size = 500; // Process 500 records at a time
|
||||||
$record_count = 0; // Initialize record counter
|
$record_count = 0; // Initialize record counter
|
||||||
|
$max_records = 50000; // Safety limit to prevent runaway processing
|
||||||
|
$start_time = time(); // Track processing time
|
||||||
|
$max_processing_time = 1200; // 20 minutes max for processing
|
||||||
|
|
||||||
while ($record = $this->adif_parser->get_record()) {
|
while ($record = $this->adif_parser->get_record()) {
|
||||||
$record_count++; // Increment counter for each record read
|
$record_count++; // Increment counter for each record read
|
||||||
|
|
||||||
|
// Safety checks to prevent runaway processing
|
||||||
|
if ($record_count > $max_records) {
|
||||||
|
log_message('error', 'QRZ download: Exceeded maximum record limit of ' . $max_records . ' records. Processing stopped.');
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ((time() - $start_time) > $max_processing_time) {
|
||||||
|
log_message('error', 'QRZ download: Exceeded maximum processing time of ' . $max_processing_time . ' seconds. Processing stopped at record ' . $record_count . '.');
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
if ((!(isset($record['app_qrzlog_qsldate']))) || (!(isset($record['qso_date'])))) {
|
if ((!(isset($record['app_qrzlog_qsldate']))) || (!(isset($record['qso_date'])))) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
@ -509,6 +534,12 @@ class Qrz extends CI_Controller {
|
||||||
if (count($batch_data) >= $batch_size) {
|
if (count($batch_data) >= $batch_size) {
|
||||||
$table .= $this->logbook_model->process_qrz_batch($batch_data);
|
$table .= $this->logbook_model->process_qrz_batch($batch_data);
|
||||||
$batch_data = []; // Reset batch
|
$batch_data = []; // Reset batch
|
||||||
|
|
||||||
|
// Log progress every 1000 records to help monitor long-running processes
|
||||||
|
if ($record_count % 1000 == 0) {
|
||||||
|
$elapsed_time = time() - $start_time;
|
||||||
|
log_message('info', 'QRZ download progress: ' . $record_count . ' records processed in ' . $elapsed_time . ' seconds.');
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -517,6 +548,10 @@ class Qrz extends CI_Controller {
|
||||||
$table .= $this->logbook_model->process_qrz_batch($batch_data);
|
$table .= $this->logbook_model->process_qrz_batch($batch_data);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Log successful completion with statistics
|
||||||
|
$processing_time = time() - $start_time;
|
||||||
|
log_message('info', 'QRZ download completed successfully. Processed ' . $record_count . ' records in ' . $processing_time . ' seconds.');
|
||||||
|
|
||||||
if ($table != "") {
|
if ($table != "") {
|
||||||
$data['tableheaders'] = $tableheaders;
|
$data['tableheaders'] = $tableheaders;
|
||||||
$data['table'] = $table;
|
$data['table'] = $table;
|
||||||
|
|
|
||||||
|
|
@ -151,7 +151,7 @@ class Labels_model extends CI_Model {
|
||||||
$this->db->where('station_profile.user_id', $this->session->userdata('user_id'));
|
$this->db->where('station_profile.user_id', $this->session->userdata('user_id'));
|
||||||
$this->db->where_in('COL_QSL_SENT', array('R', 'Q'));
|
$this->db->where_in('COL_QSL_SENT', array('R', 'Q'));
|
||||||
$this->db->order_by("COL_DXCC", "ASC");
|
$this->db->order_by("COL_DXCC", "ASC");
|
||||||
$this->db->order_by("COL_CALL", "ASC");
|
$this->db->order_by("CASE WHEN COL_QSL_VIA IS NOT NULL AND COL_QSL_VIA != '' THEN COL_QSL_VIA ELSE COL_CALL END", "ASC");
|
||||||
$this->db->order_by("COL_SAT_NAME", "ASC");
|
$this->db->order_by("COL_SAT_NAME", "ASC");
|
||||||
$this->db->order_by("COL_SAT_MODE", "ASC");
|
$this->db->order_by("COL_SAT_MODE", "ASC");
|
||||||
$this->db->order_by("COL_BAND_RX", "ASC");
|
$this->db->order_by("COL_BAND_RX", "ASC");
|
||||||
|
|
@ -169,6 +169,7 @@ class Labels_model extends CI_Model {
|
||||||
$this->db->where('station_profile.user_id', $this->session->userdata('user_id'));
|
$this->db->where('station_profile.user_id', $this->session->userdata('user_id'));
|
||||||
$this->db->where_in('COL_PRIMARY_KEY', $ids);
|
$this->db->where_in('COL_PRIMARY_KEY', $ids);
|
||||||
$this->db->order_by("COL_DXCC", "ASC");
|
$this->db->order_by("COL_DXCC", "ASC");
|
||||||
|
$this->db->order_by("CASE WHEN COL_QSL_VIA IS NOT NULL AND COL_QSL_VIA != '' THEN COL_QSL_VIA ELSE COL_CALL END", "ASC");
|
||||||
$query = $this->db->get($this->config->item('table_name'));
|
$query = $this->db->get($this->config->item('table_name'));
|
||||||
|
|
||||||
return $query;
|
return $query;
|
||||||
|
|
|
||||||
正在加载…
在新工单中引用